From 19aa1408da006e314a3d9cfe92fe7586b4387699 Mon Sep 17 00:00:00 2001 From: javi11 Date: Sat, 14 Nov 2026 10:18:23 +0100 Subject: [PATCH 1/5] feat(peernet): share and import verified release bundles Adds exposure level 3. A node can serve a sanitised, compressed description of a finished import (a bundle) and another node can mount the release from it after verifying it locally, without downloading or analysing the NZB. Bundles: - canonical, strictly decoded format with zstd compression, bounded decompression and bundle-wide budgets; fuzz tested - never contains local or store paths, rclone secrets, known holes, indexer, category, ARR or provider data - not shareable: rclone-encrypted files, files repaired by local PAR2 patches, keyed releases unless the password came inside the NZB itself Receiving: - verified in every network before anything is written: hash against the signed record, structure, no gaps, one NZB entry per file, sizes against the NZB, article presence on the user's providers, sampled yEnc headers, and a structural media check through the layout - playable at once; a full check of every segment is forced; the import is announced to ARR applications and published only after that check passes - failed or stale imports are removed and reported as failed - importer fast path for private networks; opt-in for the open network and then only for bundles announced identically by two known publishers Also: - older imports get a release identity derived from their file set, with no NZB parsing, and health evidence now matches by file - stream listings show peer releases with a fetchable bundle, imported and verified on play - settings UI for level 3, activity statistics and badges for peer imports --- cmd/altmount/cmd/peernet_bundle_wiring.go | 347 +++++++++ .../cmd/peernet_bundle_wiring_test.go | 267 +++++++ cmd/altmount/cmd/peernet_stremio_wiring.go | 73 ++ cmd/altmount/cmd/serve.go | 30 +- cmd/altmount/cmd/setup.go | 12 +- config.sample.yaml | 1 + docs/docs/3. Configuration/peer-network.md | 84 ++- .../specs/2026-11-12-peer-network-design.md | 516 ++++++++++++++ .../config/peernet/ActivityCard.tsx | 87 +++ .../config/peernet/ExposureLevelSelect.tsx | 73 +- .../config/peernet/OpenNetworkCard.tsx | 18 + .../components/config/peernet/peernetStats.ts | 179 ++++- .../config/peernet/peernetValidation.ts | 16 +- .../src/components/queue/QueueItemCard.tsx | 10 + frontend/src/components/ui/PeerBadge.tsx | 23 + .../components/HealthTable/HealthItemCard.tsx | 5 + .../components/HealthTable/HealthTableRow.tsx | 5 + frontend/src/pages/QueuePage.tsx | 9 + frontend/src/types/api.ts | 8 + frontend/src/types/peernet.ts | 50 +- frontend/tests/peernetStats.test.ts | 250 ++++++- frontend/tests/peernetValidation.test.ts | 12 + internal/api/health_handlers.go | 3 + internal/api/peer_bundle_origin.go | 71 ++ internal/api/peer_bundle_origin_test.go | 75 ++ internal/api/peernet_handlers.go | 82 ++- internal/api/peernet_handlers_test.go | 41 +- internal/api/queue_handlers.go | 29 +- internal/api/queue_password_origin_test.go | 17 + internal/api/queue_peer_import_test.go | 61 ++ internal/api/sabnzbd_handlers.go | 15 +- internal/api/sabnzbd_peer_hold_test.go | 136 ++++ internal/api/sabnzbd_types.go | 24 + internal/api/server.go | 11 +- internal/api/stremio_addon_handlers.go | 19 +- internal/api/stremio_peer.go | 393 +++++++++++ internal/api/stremio_peer_test.go | 436 ++++++++++++ internal/api/types.go | 44 +- internal/config/peernet.go | 6 + internal/config/peernet_test.go | 1 + .../057_add_peernet_release_scheme.sql | 24 + .../058_add_peernet_publish_bundle.sql | 16 + .../059_add_peernet_record_bundle.sql | 8 + .../060_add_peernet_bundle_imports.sql | 33 + .../061_add_peernet_password_origin.sql | 15 + .../062_add_import_queue_peer_hold.sql | 20 + .../sqlite/057_add_peernet_release_scheme.sql | 24 + .../sqlite/058_add_peernet_publish_bundle.sql | 16 + .../sqlite/059_add_peernet_record_bundle.sql | 8 + .../sqlite/060_add_peernet_bundle_imports.sql | 33 + .../061_add_peernet_password_origin.sql | 15 + .../sqlite/062_add_import_queue_peer_hold.sql | 20 + internal/database/models.go | 50 +- .../peernet_bundle_import_repository.go | 515 ++++++++++++++ .../peernet_bundle_import_repository_test.go | 249 +++++++ internal/database/peernet_bundle_queries.go | 171 +++++ internal/database/peernet_bundle_test.go | 176 +++++ internal/database/peernet_content_derive.go | 364 ++++++++++ .../database/peernet_content_derive_test.go | 214 ++++++ .../database/peernet_content_repository.go | 104 ++- .../peernet_content_repository_test.go | 6 +- .../peernet_password_origin_repository.go | 56 ++ ...peernet_password_origin_repository_test.go | 45 ++ .../peernet_publish_state_repository.go | 26 +- internal/database/peernet_queue_hold_test.go | 192 +++++ .../database/peernet_record_files_test.go | 158 +++++ internal/database/peernet_record_queries.go | 93 ++- .../database/peernet_record_repository.go | 50 +- internal/database/repository.go | 55 +- internal/database/testing_test.go | 1 + internal/health/checker.go | 13 +- internal/health/peer_advice.go | 4 + internal/health/peer_bundle.go | 106 +++ internal/health/peer_bundle_test.go | 211 ++++++ internal/health/worker.go | 39 ++ .../importer/ensure_persistent_nzb_test.go | 1 + internal/importer/filesystem/utils.go | 77 +- internal/importer/handle_failure_test.go | 1 + internal/importer/nzb_password_origin.go | 104 +++ internal/importer/nzb_password_origin_test.go | 99 +++ internal/importer/parser/parser.go | 43 +- internal/importer/parser/parser_test.go | 40 ++ internal/importer/parser/types.go | 16 + internal/importer/peer_bundle.go | 196 ++++++ internal/importer/peer_bundle_service.go | 278 ++++++++ internal/importer/peer_bundle_service_test.go | 240 +++++++ internal/importer/peer_bundle_test.go | 227 ++++++ .../postprocessor/health_scheduler.go | 13 +- internal/importer/processor.go | 43 +- internal/importer/service.go | 19 +- internal/metadata/service.go | 46 +- internal/peernet/bridge/bundleimport.go | 270 +++++++ internal/peernet/bridge/bundleimport_test.go | 236 +++++++ internal/peernet/bridge/nooffer_bench_test.go | 51 ++ internal/peernet/bridge/offercache.go | 39 ++ internal/peernet/bridge/offercache_test.go | 48 ++ internal/peernet/bridge/passwordorigin.go | 75 ++ .../peernet/bridge/passwordorigin_test.go | 57 ++ internal/peernet/bundle/budget_test.go | 74 ++ internal/peernet/bundle/build.go | 425 ++++++++++++ internal/peernet/bundle/build_test.go | 302 ++++++++ internal/peernet/bundle/compress.go | 108 +++ internal/peernet/bundle/compress_test.go | 166 +++++ internal/peernet/bundle/convert.go | 136 ++++ internal/peernet/bundle/convert_test.go | 203 ++++++ internal/peernet/bundle/coverage_test.go | 71 ++ internal/peernet/bundle/fuzz_test.go | 167 +++++ internal/peernet/bundle/gap_test.go | 39 ++ internal/peernet/bundle/helpers_test.go | 382 ++++++++++ internal/peernet/bundle/limits.go | 100 +++ internal/peernet/bundle/sanitize.go | 170 +++++ internal/peernet/bundle/sanitize_test.go | 127 ++++ internal/peernet/bundle/types.go | 122 ++++ internal/peernet/bundle/validate.go | 577 +++++++++++++++ internal/peernet/bundle/validate_test.go | 241 +++++++ internal/peernet/bundle/verify.go | 104 +++ internal/peernet/bundle/verify_test.go | 220 ++++++ internal/peernet/bundle/wire.go | 225 ++++++ internal/peernet/bundle/wire_decode.go | 656 ++++++++++++++++++ internal/peernet/bundle/wire_test.go | 224 ++++++ internal/peernet/bundleimport/accept.go | 440 ++++++++++++ internal/peernet/bundleimport/accept_test.go | 556 +++++++++++++++ .../peernet/bundleimport/collision_test.go | 169 +++++ internal/peernet/bundleimport/failure_test.go | 243 +++++++ internal/peernet/bundleimport/filepolicy.go | 135 ++++ .../peernet/bundleimport/filepolicy_test.go | 90 +++ .../peernet/bundleimport/helpers_pool_test.go | 29 + internal/peernet/bundleimport/importer.go | 83 +++ internal/peernet/bundleimport/pathobserver.go | 138 ++++ .../peernet/bundleimport/pathobserver_test.go | 69 ++ internal/peernet/bundleimport/reconcile.go | 107 +++ internal/peernet/bundleimport/release.go | 90 +++ internal/peernet/bundleimport/sample.go | 210 ++++++ internal/peernet/bundleimport/sizes.go | 77 ++ internal/peernet/bundleimport/sizes_test.go | 59 ++ internal/peernet/bundleimport/staging.go | 85 +++ internal/peernet/bundleimport/structure.go | 502 ++++++++++++++ .../peernet/bundleimport/structure_test.go | 80 +++ .../bundleimport/structure_variants_test.go | 231 ++++++ internal/peernet/bundleimport/tracker.go | 311 +++++++++ internal/peernet/bundleimport/tracker_test.go | 390 +++++++++++ internal/peernet/bundleimport/types.go | 418 +++++++++++ internal/peernet/bundleimport/yenc_test.go | 92 +++ internal/peernet/bundlex/buckets.go | 96 +++ internal/peernet/bundlex/build_bench_test.go | 34 + internal/peernet/bundlex/builder.go | 444 ++++++++++++ internal/peernet/bundlex/builder_test.go | 219 ++++++ internal/peernet/bundlex/dos_test.go | 96 +++ internal/peernet/bundlex/fetch.go | 581 ++++++++++++++++ internal/peernet/bundlex/fetch_test.go | 397 +++++++++++ internal/peernet/bundlex/helpers_test.go | 119 ++++ internal/peernet/bundlex/limits.go | 140 ++++ internal/peernet/bundlex/serve.go | 336 +++++++++ internal/peernet/bundlex/serve_test.go | 334 +++++++++ internal/peernet/bundlex/source.go | 142 ++++ internal/peernet/bundlex/source_test.go | 113 +++ internal/peernet/contentid/bundle_identity.go | 51 ++ .../peernet/contentid/bundle_identity_test.go | 131 ++++ internal/peernet/contentid/derive.go | 464 +++++++++++++ .../peernet/contentid/derive_bench_test.go | 80 +++ internal/peernet/contentid/derive_test.go | 488 +++++++++++++ internal/peernet/contentid/doc.go | 27 + internal/peernet/contentid/files_id.go | 64 ++ internal/peernet/contentid/files_id_test.go | 103 +++ internal/peernet/contentid/indexer.go | 79 ++- internal/peernet/contentid/indexer_test.go | 11 +- internal/peernet/contentid/observer.go | 13 + internal/peernet/contentid/reconcile.go | 25 +- internal/peernet/hints/advice.go | 219 ++++++ internal/peernet/hints/candidates.go | 137 ++++ internal/peernet/hints/files_test.go | 345 +++++++++ internal/peernet/hints/hints.go | 572 +-------------- internal/peernet/hints/hints_test.go | 38 +- internal/peernet/hints/knowncache.go | 94 +++ internal/peernet/hints/label.go | 370 ++++++++++ internal/peernet/publish/bundle_test.go | 248 +++++++ internal/peernet/publish/plan.go | 6 + internal/peernet/publish/publish.go | 104 ++- internal/peernet/publish/scheme_test.go | 74 ++ internal/peernet/records/attestation.go | 2 +- internal/peernet/records/doc.go | 10 + internal/peernet/records/envelope.go | 20 +- internal/peernet/records/fuzz_test.go | 31 + internal/peernet/records/limits.go | 7 + internal/peernet/records/release.go | 2 +- internal/peernet/records/scheme_test.go | 192 +++++ internal/peernet/records/wire.go | 67 +- .../peernet/recsync/bundle_access_test.go | 31 + internal/peernet/recsync/exchange.go | 14 + internal/peernet/recsync/ingest.go | 4 + internal/peernet/recsync/ingest_test.go | 27 + internal/peernet/recsync/policy.go | 8 +- internal/peernet/recsync/policy_test.go | 14 +- .../peernet/service/bundle_import_e2e_test.go | 279 ++++++++ internal/peernet/service/bundles.go | 324 +++++++++ internal/peernet/service/bundles_test.go | 395 +++++++++++ internal/peernet/service/derive_index_test.go | 128 ++++ .../peernet/service/frontend_labels_test.go | 82 +++ internal/peernet/service/hints.go | 162 ++++- internal/peernet/service/hints_hub_test.go | 90 ++- internal/peernet/service/index.go | 164 ++++- .../peernet/service/library_status_test.go | 12 +- internal/peernet/service/node.go | 11 + .../peernet/service/peer_candidates_test.go | 129 ++++ internal/peernet/service/peer_origin_test.go | 143 ++++ internal/peernet/service/publishing.go | 130 +++- .../service/publishing_derived_test.go | 100 +++ .../service/publishing_helpers_test.go | 4 +- internal/peernet/service/releaseinfo.go | 23 +- internal/peernet/service/service.go | 54 +- internal/peernet/service/status.go | 3 + internal/testsupport/fakepool/fakepool.go | 3 + 212 files changed, 26929 insertions(+), 861 deletions(-) create mode 100644 cmd/altmount/cmd/peernet_bundle_wiring.go create mode 100644 cmd/altmount/cmd/peernet_bundle_wiring_test.go create mode 100644 cmd/altmount/cmd/peernet_stremio_wiring.go create mode 100644 frontend/src/components/ui/PeerBadge.tsx create mode 100644 internal/api/peer_bundle_origin.go create mode 100644 internal/api/peer_bundle_origin_test.go create mode 100644 internal/api/queue_password_origin_test.go create mode 100644 internal/api/queue_peer_import_test.go create mode 100644 internal/api/sabnzbd_peer_hold_test.go create mode 100644 internal/api/stremio_peer.go create mode 100644 internal/api/stremio_peer_test.go create mode 100644 internal/database/migrations/postgres/057_add_peernet_release_scheme.sql create mode 100644 internal/database/migrations/postgres/058_add_peernet_publish_bundle.sql create mode 100644 internal/database/migrations/postgres/059_add_peernet_record_bundle.sql create mode 100644 internal/database/migrations/postgres/060_add_peernet_bundle_imports.sql create mode 100644 internal/database/migrations/postgres/061_add_peernet_password_origin.sql create mode 100644 internal/database/migrations/postgres/062_add_import_queue_peer_hold.sql create mode 100644 internal/database/migrations/sqlite/057_add_peernet_release_scheme.sql create mode 100644 internal/database/migrations/sqlite/058_add_peernet_publish_bundle.sql create mode 100644 internal/database/migrations/sqlite/059_add_peernet_record_bundle.sql create mode 100644 internal/database/migrations/sqlite/060_add_peernet_bundle_imports.sql create mode 100644 internal/database/migrations/sqlite/061_add_peernet_password_origin.sql create mode 100644 internal/database/migrations/sqlite/062_add_import_queue_peer_hold.sql create mode 100644 internal/database/peernet_bundle_import_repository.go create mode 100644 internal/database/peernet_bundle_import_repository_test.go create mode 100644 internal/database/peernet_bundle_queries.go create mode 100644 internal/database/peernet_bundle_test.go create mode 100644 internal/database/peernet_content_derive.go create mode 100644 internal/database/peernet_content_derive_test.go create mode 100644 internal/database/peernet_password_origin_repository.go create mode 100644 internal/database/peernet_password_origin_repository_test.go create mode 100644 internal/database/peernet_queue_hold_test.go create mode 100644 internal/database/peernet_record_files_test.go create mode 100644 internal/health/peer_bundle.go create mode 100644 internal/health/peer_bundle_test.go create mode 100644 internal/importer/nzb_password_origin.go create mode 100644 internal/importer/nzb_password_origin_test.go create mode 100644 internal/importer/peer_bundle.go create mode 100644 internal/importer/peer_bundle_service.go create mode 100644 internal/importer/peer_bundle_service_test.go create mode 100644 internal/importer/peer_bundle_test.go create mode 100644 internal/peernet/bridge/bundleimport.go create mode 100644 internal/peernet/bridge/bundleimport_test.go create mode 100644 internal/peernet/bridge/nooffer_bench_test.go create mode 100644 internal/peernet/bridge/offercache.go create mode 100644 internal/peernet/bridge/offercache_test.go create mode 100644 internal/peernet/bridge/passwordorigin.go create mode 100644 internal/peernet/bridge/passwordorigin_test.go create mode 100644 internal/peernet/bundle/budget_test.go create mode 100644 internal/peernet/bundle/build.go create mode 100644 internal/peernet/bundle/build_test.go create mode 100644 internal/peernet/bundle/compress.go create mode 100644 internal/peernet/bundle/compress_test.go create mode 100644 internal/peernet/bundle/convert.go create mode 100644 internal/peernet/bundle/convert_test.go create mode 100644 internal/peernet/bundle/coverage_test.go create mode 100644 internal/peernet/bundle/fuzz_test.go create mode 100644 internal/peernet/bundle/gap_test.go create mode 100644 internal/peernet/bundle/helpers_test.go create mode 100644 internal/peernet/bundle/limits.go create mode 100644 internal/peernet/bundle/sanitize.go create mode 100644 internal/peernet/bundle/sanitize_test.go create mode 100644 internal/peernet/bundle/types.go create mode 100644 internal/peernet/bundle/validate.go create mode 100644 internal/peernet/bundle/validate_test.go create mode 100644 internal/peernet/bundle/verify.go create mode 100644 internal/peernet/bundle/verify_test.go create mode 100644 internal/peernet/bundle/wire.go create mode 100644 internal/peernet/bundle/wire_decode.go create mode 100644 internal/peernet/bundle/wire_test.go create mode 100644 internal/peernet/bundleimport/accept.go create mode 100644 internal/peernet/bundleimport/accept_test.go create mode 100644 internal/peernet/bundleimport/collision_test.go create mode 100644 internal/peernet/bundleimport/failure_test.go create mode 100644 internal/peernet/bundleimport/filepolicy.go create mode 100644 internal/peernet/bundleimport/filepolicy_test.go create mode 100644 internal/peernet/bundleimport/helpers_pool_test.go create mode 100644 internal/peernet/bundleimport/importer.go create mode 100644 internal/peernet/bundleimport/pathobserver.go create mode 100644 internal/peernet/bundleimport/pathobserver_test.go create mode 100644 internal/peernet/bundleimport/reconcile.go create mode 100644 internal/peernet/bundleimport/release.go create mode 100644 internal/peernet/bundleimport/sample.go create mode 100644 internal/peernet/bundleimport/sizes.go create mode 100644 internal/peernet/bundleimport/sizes_test.go create mode 100644 internal/peernet/bundleimport/staging.go create mode 100644 internal/peernet/bundleimport/structure.go create mode 100644 internal/peernet/bundleimport/structure_test.go create mode 100644 internal/peernet/bundleimport/structure_variants_test.go create mode 100644 internal/peernet/bundleimport/tracker.go create mode 100644 internal/peernet/bundleimport/tracker_test.go create mode 100644 internal/peernet/bundleimport/types.go create mode 100644 internal/peernet/bundleimport/yenc_test.go create mode 100644 internal/peernet/bundlex/buckets.go create mode 100644 internal/peernet/bundlex/build_bench_test.go create mode 100644 internal/peernet/bundlex/builder.go create mode 100644 internal/peernet/bundlex/builder_test.go create mode 100644 internal/peernet/bundlex/dos_test.go create mode 100644 internal/peernet/bundlex/fetch.go create mode 100644 internal/peernet/bundlex/fetch_test.go create mode 100644 internal/peernet/bundlex/helpers_test.go create mode 100644 internal/peernet/bundlex/limits.go create mode 100644 internal/peernet/bundlex/serve.go create mode 100644 internal/peernet/bundlex/serve_test.go create mode 100644 internal/peernet/bundlex/source.go create mode 100644 internal/peernet/bundlex/source_test.go create mode 100644 internal/peernet/contentid/bundle_identity.go create mode 100644 internal/peernet/contentid/bundle_identity_test.go create mode 100644 internal/peernet/contentid/derive.go create mode 100644 internal/peernet/contentid/derive_bench_test.go create mode 100644 internal/peernet/contentid/derive_test.go create mode 100644 internal/peernet/contentid/files_id.go create mode 100644 internal/peernet/contentid/files_id_test.go create mode 100644 internal/peernet/hints/advice.go create mode 100644 internal/peernet/hints/candidates.go create mode 100644 internal/peernet/hints/files_test.go create mode 100644 internal/peernet/hints/knowncache.go create mode 100644 internal/peernet/hints/label.go create mode 100644 internal/peernet/publish/bundle_test.go create mode 100644 internal/peernet/publish/scheme_test.go create mode 100644 internal/peernet/records/scheme_test.go create mode 100644 internal/peernet/recsync/bundle_access_test.go create mode 100644 internal/peernet/service/bundle_import_e2e_test.go create mode 100644 internal/peernet/service/bundles.go create mode 100644 internal/peernet/service/bundles_test.go create mode 100644 internal/peernet/service/derive_index_test.go create mode 100644 internal/peernet/service/frontend_labels_test.go create mode 100644 internal/peernet/service/peer_candidates_test.go create mode 100644 internal/peernet/service/peer_origin_test.go create mode 100644 internal/peernet/service/publishing_derived_test.go diff --git a/cmd/altmount/cmd/peernet_bundle_wiring.go b/cmd/altmount/cmd/peernet_bundle_wiring.go new file mode 100644 index 000000000..1decb942e --- /dev/null +++ b/cmd/altmount/cmd/peernet_bundle_wiring.go @@ -0,0 +1,347 @@ +package cmd + +import ( + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "sync" + "sync/atomic" + "time" + + "github.com/kipsilabs/altmount/internal/api" + "github.com/kipsilabs/altmount/internal/config" + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/health" + "github.com/kipsilabs/altmount/internal/importer" + "github.com/kipsilabs/altmount/internal/importer/postprocessor" + "github.com/kipsilabs/altmount/internal/metadata" + "github.com/kipsilabs/altmount/internal/nzbfilesystem/segcache" + "github.com/kipsilabs/altmount/internal/par2repair" + "github.com/kipsilabs/altmount/internal/peernet/bridge" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/bundleimport" + "github.com/kipsilabs/altmount/internal/peernet/bundlex" + "github.com/kipsilabs/altmount/internal/peernet/service" + "github.com/kipsilabs/altmount/internal/pool" +) + +var errNoBundleFetcher = errors.New("peer network bundle fetching is not connected") + +// peernetBundleSeams are the two things the bundle import needs from the peer +// network side and that are connected later: the bundle fetcher and the +// bad-publisher reporter. Until they are set the defaults do nothing, and the +// importer fast path stays uninstalled, so the feature changes no behaviour. +type peernetBundleSeams struct { + mu sync.RWMutex + fetcher bundleimport.Fetcher + reporter bundleimport.BadBundleReporter +} + +// SetFetcher connects the bundle fetcher (the peer network service). +func (s *peernetBundleSeams) SetFetcher(f bundleimport.Fetcher) { + s.mu.Lock() + s.fetcher = f + s.mu.Unlock() +} + +// SetReporter connects the bad-publisher reporter. +func (s *peernetBundleSeams) SetReporter(r bundleimport.BadBundleReporter) { + s.mu.Lock() + s.reporter = r + s.mu.Unlock() +} + +func (s *peernetBundleSeams) hasFetcher() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.fetcher != nil +} + +// FetchBundle forwards to the connected fetcher; with none it fails. +func (s *peernetBundleSeams) FetchBundle(ctx context.Context, releaseID string) (*bundle.Bundle, bundleimport.Origin, error) { + s.mu.RLock() + f := s.fetcher + s.mu.RUnlock() + if f == nil { + return nil, bundleimport.Origin{}, errNoBundleFetcher + } + return f.FetchBundle(ctx, releaseID) +} + +// ReportBadBundle forwards to the connected reporter; with none it logs. +func (s *peernetBundleSeams) ReportBadBundle(ctx context.Context, o bundleimport.Origin, releaseID string) { + s.mu.RLock() + r := s.reporter + s.mu.RUnlock() + if r == nil { + slog.InfoContext(ctx, "A peer bundle failed its full check; no reporter is connected", "release_id", releaseID) + return + } + r.ReportBadBundle(ctx, o, releaseID) +} + +// ReportVerifiedBundle forwards to the connected reporter when it wants to +// hear about verified bundles. +func (s *peernetBundleSeams) ReportVerifiedBundle(ctx context.Context, o bundleimport.Origin, releaseID string) { + s.mu.RLock() + r := s.reporter + s.mu.RUnlock() + if v, ok := r.(bundleimport.BundleVerifiedReporter); ok { + v.ReportVerifiedBundle(ctx, o, releaseID) + } +} + +type fastPathTarget interface { + SetPeerBundleFastPath(importer.PeerBundleFastPath) +} +type peerPlayTarget interface { + SetPeerPlayback(api.PeerPlayback) +} +type pwRecorderTarget interface { + SetNZBPasswordRecorder(importer.NZBPasswordRecorder) +} +type healthGateTarget interface { + SetPeerBundleGate(health.PeerBundleGate) +} +type bundleLookupTarget interface { + SetPeerBundleLookup(api.PeerBundleLookup) +} + +// peernetBundleHost installs the bundle import (importer fast path, health +// gate, API badges) while the peer network is enabled and removes it while it +// is not, now and on every configuration change. With the feature off none of +// the hooks exists, so imports and health checks run no peer-network code and +// no query. The fast path additionally needs a connected fetcher. +type peernetBundleHost struct { + importerSvc fastPathTarget + health healthGateTarget + api bundleLookupTarget + lookup api.PeerBundleLookup + // pwTarget and pwRecorder install the hook that records where a keyed + // file's password came from; both may be nil. + pwTarget pwRecorderTarget + // play and peerPlay install the Stremio candidates and play route; both + // may be nil. + play peerPlayTarget + peerPlay api.PeerPlayback + pwRecorder importer.NZBPasswordRecorder + seams *peernetBundleSeams + // receiving says at least one network is at level 1 or above. + receiving func() bool + // anyOffers is the cached "this node holds a bundle offer" check (nil: assumed). + anyOffers func(context.Context) bool + // hasPending says peer-bundle imports still owe their full check. While it + // is true the gate stays installed even with the feature off (fail closed): + // such imports must still be force-checked and released on pass. + hasPending func() bool + // observe installs the importer's path observer on the metadata service. + observe func(metadata.FileObserver) + build func() *bundleimport.Importer + getCfg config.ConfigGetter + + mu sync.Mutex + imp *bundleimport.Importer +} + +func (h *peernetBundleHost) importer() *bundleimport.Importer { + if h.imp == nil { + h.imp = h.build() + if h.observe != nil { + h.observe(h.imp.PathObserver()) + } + go h.imp.ResumeAnnouncements(context.Background()) + go h.imp.RunHousekeeping(context.Background()) + } + return h.imp +} + +func (h *peernetBundleHost) apply(cfg *config.Config) { + h.mu.Lock() + defer h.mu.Unlock() + if h.pwTarget != nil { + // Recording is a one-row write per keyed import; with the feature off + // the importer has no hook at all and runs no extra code. + if cfg.PeernetEnabled() { + h.pwTarget.SetNZBPasswordRecorder(h.pwRecorder) + } else { + h.pwTarget.SetNZBPasswordRecorder(nil) + } + } + if h.play != nil { + // Stremio candidates need the feature and a connected fetcher. + if cfg.PeernetEnabled() && h.seams.hasFetcher() { + h.play.SetPeerPlayback(h.peerPlay) + } else { + h.play.SetPeerPlayback(nil) + } + } + if !cfg.PeernetEnabled() { + h.importerSvc.SetPeerBundleFastPath(nil) + if h.hasPending != nil && h.hasPending() { + h.health.SetPeerBundleGate(h.importer()) + h.api.SetPeerBundleLookup(h.lookup) + return + } + h.health.SetPeerBundleGate(nil) + h.api.SetPeerBundleLookup(nil) + return + } + imp := h.importer() + // Pending peer imports must be honoured even before a fetcher exists. + h.health.SetPeerBundleGate(imp) + h.api.SetPeerBundleLookup(h.lookup) + if h.seams.hasFetcher() { + h.importerSvc.SetPeerBundleFastPath(bridge.NewPeerBundleFastPath(bridge.FastPathConfig{ + Acceptor: imp, Fetcher: h.seams, Enabled: h.receiving, AnyOffers: h.anyOffers, + OpenFastPath: func() bool { return h.getCfg != nil && h.getCfg().Peernet.OpenNetwork.FastPath }, + })) + } else { + h.importerSvc.SetPeerBundleFastPath(nil) + } +} + +// Importer returns the bundle importer (nil until the feature was first +// enabled). The later Stremio flow calls ImportRelease on it, and the +// publisher asks PeerBundleStatus. +func (h *peernetBundleHost) Importer() *bundleimport.Importer { + h.mu.Lock() + defer h.mu.Unlock() + return h.imp +} + +// Refresh re-applies the current configuration, for use after a seam was +// connected. +func (h *peernetBundleHost) Refresh(cfg *config.Config) { h.apply(cfg) } + +func wirePeernetBundleImport(h *peernetBundleHost, cfg *config.Config, register func(config.ChangeCallback)) { + h.apply(cfg) + register(func(_, newCfg *config.Config) { h.apply(newCfg) }) +} + +type bundleWiringDeps struct { + Metadata *metadata.MetadataService + DB *database.DB + Importer *importer.Service + Pool pool.Manager + Cache *segcache.Source + GetConfig config.ConfigGetter + Receiving func() bool + Health healthGateTarget + API bundleLookupTarget +} + +// peernetStagingRoot is where hidden staging areas live: beside the database, +// as an absolute path, or in the system temp directory when no database path +// is configured (never relative to the working directory). +func peernetStagingRoot(dbPath string) string { + if dbPath == "" { + return filepath.Join(os.TempDir(), "altmount-peernet-staging") + } + dir, err := filepath.Abs(filepath.Dir(dbPath)) + if err != nil { + return filepath.Join(os.TempDir(), "altmount-peernet-staging") + } + return filepath.Join(dir, ".peernet-staging") +} + +// newPeernetBundleHost builds the real host over the application's services. +func newPeernetBundleHost(d bundleWiringDeps, seams *peernetBundleSeams) *peernetBundleHost { + conn, dialect := d.DB.Connection(), d.DB.Dialect() + state := database.NewPeernetBundleImportRepository(conn, dialect) + stagingRoot := peernetStagingRoot(d.GetConfig().Database.Path) + // A crash can leave staging areas behind; none is ever in use at start, and + // this runs whether or not the feature is enabled. + _ = os.RemoveAll(stagingRoot) + return &peernetBundleHost{ + importerSvc: d.Importer, health: d.Health, api: d.API, lookup: state, seams: seams, + pwTarget: d.Importer, + pwRecorder: bridge.NewNZBPasswordRecorder(d.Metadata, database.NewPeernetPasswordOriginRepository(conn, dialect)), + receiving: d.Receiving, getCfg: d.GetConfig, + anyOffers: bridge.NewOfferCache(database.NewPeernetRecordRepository(conn, dialect), database.BundleOfferGeneration), + observe: d.Metadata.SetSecondaryFileObserver, + hasPending: func() bool { + n, _, err := state.PendingCounts(context.Background(), time.Time{}) + return err != nil || n > 0 + }, + build: func() *bundleimport.Importer { + return bundleimport.New(bundleimport.Config{ + Library: d.Metadata, + Stager: bundleimport.NewFSStager(bundleimport.FSStagerDeps{Pool: d.Pool, Config: d.GetConfig, Cache: d.Cache, Root: stagingRoot}), + Prober: bridge.NewReleaseProber(d.Importer), + State: state, + Index: database.NewPeernetContentRepository(conn, dialect), + Fetcher: seams, + Recorder: bridge.NewRecorder(d.Importer), + Announcer: bridge.NewAnnouncer(d.Importer), + Reporter: seams, + HealthEnabled: func() bool { return d.GetConfig().GetHealthEnabled() }, + WillBeChecked: func(p string) bool { return postprocessor.WillScheduleHealth(d.GetConfig(), p) }, + AllowedExtensions: func() []string { return d.GetConfig().Import.AllowedFileExtensions }, + }) + }, + } +} + +// connectPeernetBundleSeams connects the peer network service as the bundle +// fetcher and the bad-publisher reporter, then re-applies the configuration so +// the importer fast path appears. With the feature off nothing is installed. +func connectPeernetBundleSeams(h *peernetBundleHost, svc bridge.BundleService, cfg *config.Config) { + h.seams.SetFetcher(bridge.NewBundleFetcher(svc)) + h.seams.SetReporter(bridge.NewBundleReporter(svc)) + h.Refresh(cfg) +} + +var errPeerOriginNotReady = errors.New("peer bundle state is not available yet") + +// peerOriginBinder is the publisher's view of the bundle importer, bound once +// the bundle host exists (the peer network service is created first). It fails +// closed: until the importer exists no file counts as verified, so nothing is +// published on an unanswerable question. +type peerOriginBinder struct { + h atomic.Pointer[peernetBundleHost] +} + +// Bind connects the bundle host. +func (b *peerOriginBinder) Bind(h *peernetBundleHost) { b.h.Store(h) } + +// PeerBundleStatus implements service.PeerBundleOrigin. +func (b *peerOriginBinder) PeerBundleStatus(ctx context.Context, filePath string) (fromPeer, fullCheckPassed bool, err error) { + h := b.h.Load() + if h == nil { + return false, false, errPeerOriginNotReady + } + imp := h.Importer() + if imp == nil { + return false, false, errPeerOriginNotReady + } + return imp.PeerBundleStatus(ctx, filePath) +} + +// peernetPatches gives the bundle builder its PAR2 patch index. A node with no +// patch store shares as one with an empty store. +func peernetPatches(store *par2repair.PatchStore) bundlex.PatchIndex { + if store == nil { + return bundlex.NoPatches{} + } + return store +} + +// ImportStatus maps the importer's counters for the peer network status (zero +// while no importer exists). +func (b *peerOriginBinder) ImportStatus() service.BundleImportStatus { + h := b.h.Load() + if h == nil || h.Importer() == nil { + return service.BundleImportStatus{} + } + c := h.Importer().Counters() + rej := make(map[string]int64, len(c.Rejected)) + for k, v := range c.Rejected { + rej[string(k)] = int64(v) + } + pending, stale := h.Importer().PendingStatus(context.Background()) + return service.BundleImportStatus{Pending: int64(pending), Stale: int64(stale), Accepted: int64(c.Accepted), AlreadyImported: int64(c.AlreadyImported), Rejected: rej, + FullCheckPassed: int64(c.FullCheckPassed), FullCheckFailed: int64(c.FullCheckFailed), + Announced: int64(c.Announced), BadBundleReports: int64(c.BadBundleReports)} +} diff --git a/cmd/altmount/cmd/peernet_bundle_wiring_test.go b/cmd/altmount/cmd/peernet_bundle_wiring_test.go new file mode 100644 index 000000000..637a029dc --- /dev/null +++ b/cmd/altmount/cmd/peernet_bundle_wiring_test.go @@ -0,0 +1,267 @@ +package cmd + +import ( + "context" + "path/filepath" + "testing" + + "github.com/kipsilabs/altmount/internal/api" + "github.com/kipsilabs/altmount/internal/config" + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/health" + "github.com/kipsilabs/altmount/internal/importer" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/bundleimport" + "github.com/kipsilabs/altmount/internal/peernet/service" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type recFastPath struct{ cur importer.PeerBundleFastPath } + +func (r *recFastPath) SetPeerBundleFastPath(h importer.PeerBundleFastPath) { r.cur = h } + +type recGate struct{ cur health.PeerBundleGate } + +func (r *recGate) SetPeerBundleGate(g health.PeerBundleGate) { r.cur = g } + +type recLookup struct{ cur api.PeerBundleLookup } + +func (r *recLookup) SetPeerBundleLookup(l api.PeerBundleLookup) { r.cur = l } + +type okFetcher struct{} + +func (okFetcher) FetchBundle(context.Context, string) (*bundle.Bundle, bundleimport.Origin, error) { + return nil, bundleimport.Origin{}, nil +} + +func bundleHostFixture(t *testing.T) (*peernetBundleHost, *recFastPath, *recGate, *recLookup, *int) { + t.Helper() + fp, g, l, built := &recFastPath{}, &recGate{}, &recLookup{}, new(int) + h := &peernetBundleHost{ + importerSvc: fp, health: g, api: l, seams: &peernetBundleSeams{}, receiving: func() bool { return true }, + build: func() *bundleimport.Importer { *built++; return bundleimport.New(bundleimport.Config{}) }, + } + return h, fp, g, l, built +} + +func enabledCfg(on bool) *config.Config { + cfg := config.DefaultConfig() + cfg.Peernet.Enabled = &on + return cfg +} + +func TestBundleHost_FeatureOffInstallsNothingAndBuildsNothing(t *testing.T) { + h, fp, g, l, built := bundleHostFixture(t) + h.apply(enabledCfg(false)) + assert.Nil(t, fp.cur) + assert.Nil(t, g.cur) + assert.Nil(t, l.cur) + assert.Zero(t, *built, "no importer, no database access, nothing") + assert.Nil(t, h.Importer()) +} + +func TestBundleHost_EnabledWithoutFetcherInstallsGateAndBadgesOnly(t *testing.T) { + h, fp, g, l, built := bundleHostFixture(t) + h.lookup = nil + h.apply(enabledCfg(true)) + assert.NotNil(t, g.cur, "pending peer imports are honoured even before a fetcher exists") + assert.Nil(t, fp.cur, "no fast path without a fetcher") + _ = l + assert.Equal(t, 1, *built) +} + +func TestBundleHost_FetcherInstallsFastPathAndDisableRemovesEverything(t *testing.T) { + h, fp, g, l, built := bundleHostFixture(t) + h.seams.SetFetcher(okFetcher{}) + h.apply(enabledCfg(true)) + require.NotNil(t, fp.cur) + assert.NotNil(t, g.cur) + + h.apply(enabledCfg(true)) + assert.Equal(t, 1, *built, "the importer is built once and reused") + + h.apply(enabledCfg(false)) + assert.Nil(t, fp.cur) + assert.Nil(t, g.cur) + assert.Nil(t, l.cur) +} + +func TestBundleSeams_DefaultsAreNoOps(t *testing.T) { + s := &peernetBundleSeams{} + _, _, err := s.FetchBundle(context.Background(), "x") + assert.ErrorIs(t, err, errNoBundleFetcher) + s.ReportBadBundle(context.Background(), bundleimport.Origin{}, "x") // must not panic +} + +type fakeBundleSvc struct{ noted []string } + +func (fakeBundleSvc) FetchBundle(context.Context, string) (*bundle.Bundle, service.FetchSource, error) { + return nil, service.FetchSource{}, nil +} +func (f *fakeBundleSvc) NoteBundleFullCheck(n, p string, passed bool) { + f.noted = append(f.noted, n+"/"+p+"/"+map[bool]string{true: "pass", false: "fail"}[passed]) +} + +func TestConnectBundleSeams_FastPathOnlyWhenFeatureEnabled(t *testing.T) { + h, fp, g, _, built := bundleHostFixture(t) + svc := &fakeBundleSvc{} + connectPeernetBundleSeams(h, svc, enabledCfg(false)) + assert.Nil(t, fp.cur, "feature off: nothing installed") + assert.Nil(t, g.cur) + assert.Zero(t, *built) + + connectPeernetBundleSeams(h, svc, enabledCfg(true)) + require.NotNil(t, fp.cur, "feature on and fetcher connected") + assert.NotNil(t, g.cur) + + h.seams.ReportBadBundle(context.Background(), bundleimport.Origin{NetworkID: "n", PublisherID: "p"}, "r") + h.seams.ReportVerifiedBundle(context.Background(), bundleimport.Origin{NetworkID: "n", PublisherID: "p"}, "r") + assert.Equal(t, []string{"n/p/fail", "n/p/pass"}, svc.noted) +} + +type stubBundleState struct{ bundleimport.State } + +func (stubBundleState) Get(_ context.Context, p string) (database.PeernetBundleImport, bool, error) { + if p == "peer.mkv" { + return database.PeernetBundleImport{State: "pending"}, true, nil + } + return database.PeernetBundleImport{}, false, nil +} + +func TestPeerOriginBinder_FailsClosedUntilTheImporterExists(t *testing.T) { + b := &peerOriginBinder{} + _, _, err := b.PeerBundleStatus(context.Background(), "peer.mkv") + require.Error(t, err, "unbound") + + h, _, _, _, _ := bundleHostFixture(t) + h.build = func() *bundleimport.Importer { return bundleimport.New(bundleimport.Config{State: stubBundleState{}}) } + b.Bind(h) + _, _, err = b.PeerBundleStatus(context.Background(), "peer.mkv") + require.Error(t, err, "importer not built yet (feature not applied)") + + h.apply(enabledCfg(true)) + from, passed, err := b.PeerBundleStatus(context.Background(), "peer.mkv") + require.NoError(t, err) + assert.True(t, from) + assert.False(t, passed) + from, _, err = b.PeerBundleStatus(context.Background(), "local.mkv") + require.NoError(t, err) + assert.False(t, from) +} + +func TestPeernetPatches_NilStoreMeansNoPatches(t *testing.T) { + require.NotNil(t, peernetPatches(nil)) + assert.False(t, peernetPatches(nil).Has("x@y")) +} + +func TestPeerOriginBinder_ImportStatusMapsCounters(t *testing.T) { + b := &peerOriginBinder{} + assert.Equal(t, service.BundleImportStatus{}, b.ImportStatus(), "unbound: zero") + h, _, _, _, _ := bundleHostFixture(t) + b.Bind(h) + h.apply(enabledCfg(true)) + st := b.ImportStatus() + assert.NotNil(t, st.Rejected) + assert.Zero(t, st.Accepted) +} + +type recPwTarget struct{ cur importer.NZBPasswordRecorder } + +func (r *recPwTarget) SetNZBPasswordRecorder(x importer.NZBPasswordRecorder) { r.cur = x } + +type noopPwRecorder struct{} + +func (noopPwRecorder) RecordNZBPasswordFiles(context.Context, []string) {} + +func TestBundleHost_PasswordRecorderOnlyWhileTheFeatureIsOn(t *testing.T) { + h, _, _, _, _ := bundleHostFixture(t) + target := &recPwTarget{} + h.pwTarget, h.pwRecorder = target, noopPwRecorder{} + h.apply(enabledCfg(false)) + assert.Nil(t, target.cur, "feature off: the importer has no hook") + h.apply(enabledCfg(true)) + assert.NotNil(t, target.cur) + h.apply(enabledCfg(false)) + assert.Nil(t, target.cur) +} + +type recPlayTarget struct{ cur api.PeerPlayback } + +func (r *recPlayTarget) SetPeerPlayback(p api.PeerPlayback) { r.cur = p } + +type nopPlayback struct{} + +func (nopPlayback) Candidates(context.Context, string, *int, *int) []service.PeerCandidate { + return nil +} +func (nopPlayback) HasOffer(context.Context, string) bool { return false } +func (nopPlayback) Play(context.Context, api.PeerPlayRequest) (api.PeerPlayResult, error) { + return api.PeerPlayResult{}, nil +} + +func TestBundleHost_StremioPlaybackNeedsTheFeatureAndAFetcher(t *testing.T) { + h, _, _, _, _ := bundleHostFixture(t) + target := &recPlayTarget{} + h.play, h.peerPlay = target, nopPlayback{} + h.apply(enabledCfg(true)) + assert.Nil(t, target.cur, "no fetcher: nothing to play from") + h.seams.SetFetcher(okFetcher{}) + h.apply(enabledCfg(true)) + assert.NotNil(t, target.cur) + h.apply(enabledCfg(false)) + assert.Nil(t, target.cur, "feature off: the stream list and the route are as before") +} + +type fakePeerSvc struct{ offers []service.Offer } + +func (fakePeerSvc) PeerCandidates(context.Context, string, *int, *int) []service.PeerCandidate { + return nil +} +func (f fakePeerSvc) BundleOffers(context.Context, string) ([]service.Offer, error) { + return f.offers, nil +} + +type noImporter struct{} + +func (noImporter) Importer() *bundleimport.Importer { return nil } + +type dirsFake struct{} + +func (dirsFake) PeerImportDir(c, title string) string { return "/complete/" + c + "/" + title } + +func TestStremioPeerPlayback_OffersAndNotReady(t *testing.T) { + p := stremioPeerPlayback{svc: fakePeerSvc{}, host: noImporter{}, dirs: dirsFake{}} + assert.False(t, p.HasOffer(context.Background(), "x")) + p.svc = fakePeerSvc{offers: []service.Offer{{Own: true, Fetchable: true}, {Fetchable: false}}} + assert.False(t, p.HasOffer(context.Background(), "x"), "own or undialable offers do not count") + p.svc = fakePeerSvc{offers: []service.Offer{{Fetchable: true}}} + assert.True(t, p.HasOffer(context.Background(), "x")) + _, err := p.Play(context.Background(), api.PeerPlayRequest{ReleaseID: "x"}) + assert.ErrorIs(t, err, errPeerImporterNotReady) +} + +func TestBundleHost_FeatureOffKeepsTheGateWhilePeerImportsArePending(t *testing.T) { + h, fp, g, _, built := bundleHostFixture(t) + pending := true + h.hasPending = func() bool { return pending } + h.seams.SetFetcher(okFetcher{}) + h.apply(enabledCfg(false)) + assert.NotNil(t, g.cur, "pending imports still get their forced full check and the ARR hold is released on pass") + assert.Nil(t, fp.cur, "no new fast-path imports while the feature is off") + assert.Equal(t, 1, *built) + + pending = false + h.apply(enabledCfg(false)) + assert.Nil(t, g.cur, "nothing pending: the gate is removed again") +} + +func TestPeernetStagingRoot_IsAlwaysAbsoluteAndNeverTheWorkingDirectory(t *testing.T) { + for _, dbPath := range []string{"", "altmount.db", "data/altmount.db", "/var/lib/altmount/altmount.db"} { + root := peernetStagingRoot(dbPath) + assert.True(t, filepath.IsAbs(root), "%q gave %q", dbPath, root) + assert.NotEqual(t, ".peernet-staging", root) + assert.Contains(t, filepath.Base(root), "peernet-staging") + } + assert.Equal(t, "/var/lib/altmount/.peernet-staging", peernetStagingRoot("/var/lib/altmount/altmount.db")) +} diff --git a/cmd/altmount/cmd/peernet_stremio_wiring.go b/cmd/altmount/cmd/peernet_stremio_wiring.go new file mode 100644 index 000000000..006ab18ca --- /dev/null +++ b/cmd/altmount/cmd/peernet_stremio_wiring.go @@ -0,0 +1,73 @@ +package cmd + +import ( + "context" + "errors" + + "github.com/kipsilabs/altmount/internal/api" + "github.com/kipsilabs/altmount/internal/peernet/bundleimport" + "github.com/kipsilabs/altmount/internal/peernet/service" +) + +var errPeerImporterNotReady = errors.New("peer bundle importer is not available") + +// peerCandidateService is the part of the peer network service the Stremio +// support needs. +type peerCandidateService interface { + PeerCandidates(ctx context.Context, imdbID string, season, episode *int) []service.PeerCandidate + BundleOffers(ctx context.Context, releaseID string) ([]service.Offer, error) +} + +// importerProvider returns the bundle importer (nil until the feature was +// enabled); *peernetBundleHost has this shape. +type importerProvider interface { + Importer() *bundleimport.Importer +} + +// peerDirs says where a peer import of a category and title is written. +type peerDirs interface { + PeerImportDir(category, title string) string +} + +// stremioPeerPlayback is the Stremio addon's view of the peer network: the +// candidates for a media key, whether a release can be fetched at all, and the +// import of one release into the folder a normal import of its category uses. +type stremioPeerPlayback struct { + svc peerCandidateService + host importerProvider + dirs peerDirs +} + +func (p stremioPeerPlayback) Candidates(ctx context.Context, imdbID string, season, episode *int) []service.PeerCandidate { + return p.svc.PeerCandidates(ctx, imdbID, season, episode) +} + +// HasOffer reports whether a stored signed release record offers a bundle of +// the release from a peer this node can dial. It reads the local store only. +func (p stremioPeerPlayback) HasOffer(ctx context.Context, releaseID string) bool { + offers, err := p.svc.BundleOffers(ctx, releaseID) + if err != nil { + return false + } + for _, o := range offers { + if o.Fetchable && !o.Own { + return true + } + } + return false +} + +func (p stremioPeerPlayback) Play(ctx context.Context, req api.PeerPlayRequest) (api.PeerPlayResult, error) { + imp := p.host.Importer() + if imp == nil { + return api.PeerPlayResult{}, errPeerImporterNotReady + } + res, err := imp.ImportRelease(ctx, req.ReleaseID, bundleimport.Target{ + Category: req.Category, Metadata: req.Metadata, DownloadID: req.DownloadID, + DirFunc: func(title string) string { return p.dirs.PeerImportDir(req.Category, title) }, + }) + if err != nil { + return api.PeerPlayResult{}, err + } + return api.PeerPlayResult{Paths: res.Paths, Sizes: res.Sizes}, nil +} diff --git a/cmd/altmount/cmd/serve.go b/cmd/altmount/cmd/serve.go index 60866c458..cacde21f5 100644 --- a/cmd/altmount/cmd/serve.go +++ b/cmd/altmount/cmd/serve.go @@ -23,6 +23,7 @@ import ( "github.com/kipsilabs/altmount/internal/importer" "github.com/kipsilabs/altmount/internal/metadata" "github.com/kipsilabs/altmount/internal/nzbfilesystem/segcache" + "github.com/kipsilabs/altmount/internal/peernet/service" "github.com/kipsilabs/altmount/internal/pool" "github.com/kipsilabs/altmount/internal/progress" "github.com/kipsilabs/altmount/internal/rclone" @@ -194,7 +195,16 @@ func runServe(cmd *cobra.Command, args []string) error { stremioCleanup.StartCleanup(ctx) apiServer := setupAPIServer(app, repos, authService, configManager, metadataReader, metadataService, fs, poolManager, importerService, arrsService, mountService, progressBroadcaster, streamTracker, cacheSource) - peernetService := startPeernetService(ctx, cfg, db, metadataService, repos.HealthRepo, configManager) + // The PAR2 patch index lets level 3 prove a release is unpatched; the + // origin binder (bound to the bundle host below) keeps peer-imported files + // from being published before their full check passed. + peerOrigin := &peerOriginBinder{} + peernetService := startPeernetService(ctx, cfg, db, metadataService, repos.HealthRepo, configManager, + func(d *service.Deps) { + d.Patches = peernetPatches(par2RepairService.PatchStore()) + d.PeerOrigin = peerOrigin + d.BundleImportStatus = peerOrigin.ImportStatus + }) apiServer.SetPeernetService(peernetService) apiServer.SetLogFilePath(slogutil.GetLogFilePath(cfg.Log)) apiServer.SetMigrationRepo(db.MigrationRepo) @@ -252,6 +262,24 @@ func runServe(cmd *cobra.Command, args []string) error { wirePeernetHealthObserver(healthWorker, cfg, peernetService, configManager.OnConfigChange) // Peer evidence for scheduled checks (nil adviser while the feature is off). wirePeernetHints(ctx, peernetService, healthWorker, apiServer, db, configManager, cfg) + // Bundle import: fast path, full-check gate and queue/health badges (all + // absent while the feature is off). The fetcher and the bad-publisher + // reporter are connected through the seams (peernetBundleSeams). + bundleHost := newPeernetBundleHost(bundleWiringDeps{ + Metadata: metadataService, DB: db, Importer: importerService, Pool: poolManager, Cache: cacheSource, + GetConfig: configManager.GetConfigGetter(), Health: healthWorker, API: apiServer, + Receiving: func() bool { return len(peernetService.ReceivingNetworks()) > 0 }, + }, &peernetBundleSeams{}) + // Stremio peer candidates and their play route (installed only while the + // feature is on and a fetcher is connected). + bundleHost.play = apiServer + bundleHost.peerPlay = stremioPeerPlayback{svc: peernetService, host: bundleHost, dirs: importerService} + wirePeernetBundleImport(bundleHost, cfg, configManager.OnConfigChange) + peerOrigin.Bind(bundleHost) + // Connect the halves: the service is the bundle fetcher and the + // bad-publisher reporter; the fast path installs only while the feature + // is enabled. + connectPeernetBundleSeams(bundleHost, peernetService, cfg) apiServer.SetHealthWorker(healthWorker) } if par2RepairService != nil { diff --git a/cmd/altmount/cmd/setup.go b/cmd/altmount/cmd/setup.go index adf8a6d9a..89df350fd 100644 --- a/cmd/altmount/cmd/setup.go +++ b/cmd/altmount/cmd/setup.go @@ -27,6 +27,7 @@ import ( "github.com/kipsilabs/altmount/internal/nzbfilesystem" "github.com/kipsilabs/altmount/internal/nzbfilesystem/segcache" "github.com/kipsilabs/altmount/internal/par2repair" + "github.com/kipsilabs/altmount/internal/peernet/bundlex" "github.com/kipsilabs/altmount/internal/peernet/service" "github.com/kipsilabs/altmount/internal/pool" "github.com/kipsilabs/altmount/internal/progress" @@ -468,8 +469,13 @@ func startPeernetService( metadataService *metadata.MetadataService, healthRepo *database.HealthRepository, configManager *config.Manager, + opts ...func(*service.Deps), ) *service.Service { - svc := service.New(peernetDeps(ctx, cfg, db, metadataService, healthRepo, configManager)) + deps := peernetDeps(ctx, cfg, db, metadataService, healthRepo, configManager) + for _, opt := range opts { + opt(&deps) + } + svc := service.New(deps) svc.Start(ctx) configManager.OnConfigChange(svc.OnConfigChange) return svc @@ -505,6 +511,10 @@ func peernetDeps( Exclusions: peernetExclusions(configManager.GetConfigGetter()), Backbones: newPeernetBackboneSource(configManager.GetConfigGetter(), providerbackbone.Default, nil), Metadata: service.NewMetadataHost(metadataService), + // Level 3 reads full file metadata through this source only; whether a + // keyed file's password came with its NZB is answered from the origin + // the importer recorded. + BundleSource: bundlex.NewMetadataSource(metadataService, database.NewPeernetPasswordOriginRepository(conn, d)), Health: service.NewLocalHealth(healthRepo), Settings: service.SettingsFromConfig(cfg), } diff --git a/config.sample.yaml b/config.sample.yaml index 5e5f14478..f2f7e15cd 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -322,6 +322,7 @@ peernet: open_network: level: 0 # 0 off, 1 receive only, 2 share metadata, 3 share bundles peers: [] # Peer addresses as host:port (no scheme, userinfo or path), max 64 + fast_path: false # Use layouts offered by open-network peers to skip archive analysis when you import an NZB; verified by sampling, not full analysis (default: false; private networks always may) sharing: excluded_categories: [] # Releases in these categories are never published excluded_indexers: [] # Releases from these indexers are never published diff --git a/docs/docs/3. Configuration/peer-network.md b/docs/docs/3. Configuration/peer-network.md index 48ddf0100..912cb58fa 100644 --- a/docs/docs/3. Configuration/peer-network.md +++ b/docs/docs/3. Configuration/peer-network.md @@ -21,6 +21,7 @@ peernet: open_network: level: 0 peers: [] + fast_path: false sharing: excluded_categories: [] excluded_indexers: [] @@ -33,6 +34,7 @@ peernet: - **`advertise_address`** — optional `host:port` that other peers should use to reach you. A private or LAN address is only announced on private networks that allow private addresses; the open network never uses one, and a loopback address is never usable. The settings screen warns when the address cannot be used. - **`open_network.level`** — exposure level on the open network (see below). - **`open_network.peers`** — peer addresses you enter, as `host:port` with no scheme, userinfo or path (at most 64). Loopback addresses (`127.0.0.1`, `localhost`) are rejected because they can never work; private (LAN) addresses are saved but never dialled on the open network, and the list marks them with a warning. The open network has no shipped peers; a fresh install knows nobody until you add one. +- **`open_network.fast_path`** — default `false`. Use layouts offered by open-network peers to skip archive analysis when you import an NZB. The result is verified by sampling, not by full analysis, so it is off unless you turn it on, and even then it needs at least two publishers in different network ranges announcing the identical bundle. Layouts from your private networks need no switch. Playing a peer candidate in Stremio is unaffected by this switch. - **`sharing.excluded_categories` / `sharing.excluded_indexers`** — releases in these categories or from these indexers are never published. Matching ignores case and surrounding spaces. - **`min_corroboration`** — how many distinct nodes must attest a release before it is shown as healthy on the open network (default 3, minimum 1). Nodes in the same network range count as one (see below). An explicit value you saved earlier is kept. @@ -91,7 +93,7 @@ At level 2 a peer sees which releases you have imported and found healthy, when Evidence from the network can only make a *routine* check of a file that is already healthy cheaper or later. It never decides a verdict, and a file's status is only ever set by your own checks. -- An attestation counts only when it is recent (under 7 days old), is for exactly the same file (matching is by identifiers derived from the article IDs, never by title) and comes from a node whose providers' backbones are all among yours. +- An attestation counts only when it is recent (under 7 days old), is for exactly the same file (matching is by the file's identifier derived from its article IDs, never by title, and whatever release the publisher filed the file under) and comes from a node whose providers' backbones are all among yours. - A healthy attestation also has to say how much of the file was checked. It counts only when that is at least 10 segments and at least 5% of the file's segments, which is what a normal scheduled check covers. Attestations that do not say (older ones) or that cover less never count as healthy evidence. - **"Known" nodes.** On the open network a node counts only when your instance has actually connected to it and verified its identity at least 24 hours ago, and has reached it again within the last 7 days. Nodes learned only from other peers' lists are not known until you have connected to them yourself. A node also needs an address you verified it at: on the open network at most one node per network range (/24 for IPv4, /48 for IPv6) of that address is counted, so many identities behind one range still count as one. - **Open network**: when enough distinct nodes agree (see `min_corroboration`), and each has been known to your instance for at least 24 hours, a routine check samples fewer segments (never fewer than ten, and never fewer than the first and last segments the checker always probes). The check still runs, and the next check is scheduled as usual. A check shortened this way is weaker than a normal one, so it is never published as evidence itself, and at most one shortened check happens between two normal checks of the same file. @@ -106,13 +108,24 @@ On the open network a release is shown as healthy on the network, and a check is ### Stremio labels and media-ID discovery -In Stremio, a stream served from your own library is labelled "Healthy on the peer network" when the exact release it belongs to is attested by enough nodes. The label text is fixed; nothing a peer sent is shown. Search results from your indexers are not labelled: they are NZB files that have not been downloaded yet, so their identifiers are not known, and AltMount does not guess by title. +In Stremio, a stream served from your own library is labelled "Healthy on the peer network" when every video file of the release it belongs to is attested healthy by enough nodes (matched by file, so it works for older imports too). The label text is fixed; nothing a peer sent is shown. Search results from your indexers are not labelled: they are NZB files that have not been downloaded yet, so their identifiers are not known, and AltMount does not guess by title. -Release records carry media IDs only when the publisher imported the release through Stremio, so discovery of releases by IMDb ID only covers Stremio-originated imports. In this version a release record carries no bundle, so it cannot be played or imported from the record alone and is not offered as a stream. +Release records carry media IDs only when the publisher imported the release through Stremio, so discovery of releases by IMDb ID only covers Stremio-originated imports. A release record only becomes playable when its publisher serves a bundle; see "Stremio peer candidates" below. ### Which library files can be shared -A file can be shared when your instance knows the whole release it came from, which means the release's NZB file list was kept at import time. Files imported before that was kept, or whose list was rebuilt from the surviving files by a metadata migration, still get a File ID but no Release ID, are skipped as "Release could not be identified", and are never guessed from partial data. The Activity card shows "N of M library files can be shared; the rest were imported before sharing data was kept", and `GET /peernet/status` reports it as `library` with `files`, `with_release_id`, `file_id_only` and `not_indexable`. Re-importing a release makes its files shareable. +Every library file that was imported with the release's NZB kept has a Release ID taken from that NZB. Files imported before that (the older metadata format keeps the segments inside each file) or whose list was rebuilt by a metadata migration have a File ID but not the release's full segment list, and your NZB files are **not** read again to get one. Instead, AltMount identifies such a release **by the set of files it contains**: the files that came from one import (found through the source NZB name stored in each file's metadata, never by folder, because a folder can hold several imports and one import can span folders) are given one Release ID that is the hash of their File IDs. The step runs in the background from the index AltMount already keeps, so it costs no extra reads of your files, and it runs again shortly after files are added, removed or renamed. + +What that means for you: + +- Older imports are now shared like any other: a release record and, after a healthy check, an attestation. Records say which kind of Release ID they carry, so a receiver never mistakes one for the other. +- If a file is added to or removed from such a release later, its Release ID changes. The records published under the old ID are withdrawn from your own store and the new ID is announced after the next healthy check. +- A file that has an NZB-based Release ID never also gets a derived one. +- Two nodes that imported the same NZB and kept the same files derive the same Release ID. If they kept **different subsets** of the files, their Release IDs differ. Health evidence is still shared per file: an attestation is matched by File ID whatever Release ID it was published under, so both nodes benefit from each other's checks, but the two nodes do not share the release's identity. A release is also labelled "healthy on the peer network" only when every one of its video files has qualifying evidence. +- A file whose metadata names no source NZB cannot be grouped and stays unshared; so does an import with more than 2,000 files (it gets no release ID and is counted as too large). +- The same NZB imported twice into two folders is one release: duplicate files count once. + +The Activity card shows, in plain language, how many library files can be shared, how many of those are older imports identified by their files, and why the others cannot be. `GET /peernet/status` reports this as `library` with `files`, `with_release_id`, `with_nzb_release_id`, `with_derived_release_id`, `file_id_only`, `not_indexable`, `no_source_nzb`, `group_too_large` and `pending`. A release without import history is titled from its NZB name, without a trailing media extension such as `.mkv`. @@ -120,6 +133,69 @@ A release without import history is titled from its NZB name, without a trailing Files older than 30 days are checked about every 90 days, and an attestation is only useful for 7 days. Most of an old library therefore has no fresh evidence at any moment, and the main saving is on recently added files. +## What level 3 does + +At level 3 this node serves **finished imports on request**. When a release is published, AltMount builds a *bundle* from the import it already did: the segment list (the article IDs), the file layout, sizes, offsets, nested archive sources and the original file names. Another node that has your signed release record can ask you for that bundle. **Article IDs leave your node at this level.** Level 2 shares none. + +A bundle is small (about 34 KB for a 1 GB release, a few MB for a very large one), compressed, and its hash is carried in your signed release record, so nobody can swap in a different one. Your node builds it again whenever it is asked (and keeps the last few in memory for ten minutes), and only serves bundles it announced itself and only to authenticated peers, within per-peer and node-wide rate and byte limits. + +### What can and cannot be shared as a bundle + +A release is shared as a bundle only when AltMount can prove that nothing private or unreproducible goes with it. A release that cannot be shared still publishes its normal record (without a bundle), and the status page counts the reasons. Not shareable: + +- **rclone-encrypted** files and any other encryption type (the rclone password and salt are never shared); +- releases where a **PAR2 repair** patched any article locally (the patched bytes exist only on your node), and releases whose patch status is not known; +- releases with **known holes**, unresolved segments or inconsistent local metadata; +- **keyed releases** (encrypted RAR or 7-Zip archives) whose password did not come with the NZB's own content. AltMount records, for **new imports only**, whether an archive password came from a `` that was already inside the NZB as it was received. A password typed into AltMount (the link `&p=` or the password field), spelled in the NZB's file name or in a subject line, found inside another release's archive, or supplied before this was recorded does not count, and the release is then not shared. When it does count, the bundle carries the archive key and IV (a receiver needs only those to play the release) and not the password text, which AltMount never stores. + +### What receiving a bundle does + +At level 1 and above, importing an NZB first asks the network for a bundle of exactly that release (the Release ID of the NZB's segment list), with a short time limit, and falls back to a normal import on any problem. The Stremio flow below imports a release the same way. A received bundle is untrusted. AltMount: + +1. checks its hash against the signed record and recomputes every identifier from the segment lists; +2. validates the layout (ranges, offsets, sizes, names) and applies local limits; +3. checks that the articles exist on **your own providers** (a sample probe); +4. reads the start of the main media file through the same reader that will serve it (a wrong key or wrong segments fail here); +5. only then writes the files, marked as coming from the peer network. They are **playable immediately**. + +A **full check** of every segment is then scheduled at high priority. Until it passes, the import is **not announced to the ARRs and is not published** to the network (no record, attestation or bundle). + +**What Sonarr and Radarr see while it is pending.** An item they grabbed through the SABnzbd-compatible API stays in the *queue* as work in progress (status `Downloading`, 99 percent) and is not in the *history*, so they never import content that is not verified yet. When the full check passes, the item moves to the history as completed and the ARR imports it as usual. When it fails, or when the 7-day limit below is reached, it appears in the history as **failed** with a short message ("Peer-supplied release failed its full health check and was removed"), so the ARR blocklists the release and searches again. An item you hid from the ARRs yourself (skip ARR notification) stays hidden in every state. The Queue page of AltMount shows the item as completed from the start, because the files are playable at once. + +**When a full check fails.** In every case the same single path runs and no ARR repair is ever attempted for the file: the queue item turns into a failed one, and every file of that import is removed (metadata, content index, library copy, health record and peer state), so a corrupt file does not stay in your library and the rest of the release cannot complete the import later. This holds for an item an ARR grabbed, for a release started from Stremio (which no ARR knows) and for an import hidden from the ARRs. Only files of a peer import that is in the pending state are ever removed; nothing else is touched. The publisher is **not trusted for bundle fetches for a while**: 30 minutes, doubling per repeated failure up to 6 hours, forgiven after a day or as soon as a later bundle from the same publisher passes its full check. This never affects record sync. It is mild on purpose: a failure can also mean your own providers lack articles. + +**Stale imports.** A pending import is held for at most 7 days: after that it is failed exactly as above (the publisher is not blamed, since no check ran), and the status page shows how many are pending and how many waited too long. Turning the peer network off does not strand pending imports: the full check keeps running for them until they finish. + +**Deleting and restarting.** Deleting a file removes its pending state; deleting the queue item of a pending import is allowed and the verification carries on without announcing it; if every file of an import is deleted the hold is lifted. After a restart AltMount reconciles the state: a write that a crash interrupted (a release only partly written) is rolled back completely, so it can never be announced, published or served; rows whose files no longer exist are dropped (a release with no files left is never announced); a hold nothing can release is lifted. A queue row of a release started from Stremio has no NZB: retrying, restarting or downloading the NZB of such an item is refused with a clear message, and a restart never tries to import it again. Renaming a file or folder keeps the pending state. + +The release identity the bundle proved is kept for the imported files, so they can be matched to network evidence and re-shared like local imports. If you later delete a file of a release identified by its file set, its identity changes, exactly as for a local import. + +Before anything is written the layout is also checked against the articles themselves and against the NZB: + +- **No gaps.** A bundle file may not contain a hole; a release with known holes is not shareable and a received bundle with one is refused. +- **Coverage.** In a bundle for an NZB, every file (and every archive volume or PAR2 file) must be exactly one NZB file entry's segments, in NZB order; no segment is used twice and none is left out. +- **Sizes against your NZB.** When you import an NZB, each declared decoded segment size must fit the NZB's (encoded, a few percent larger) size. +- **Sampled article headers.** For each file and archive volume the first and last article and a few random ones (chosen locally, up to 24 in all) are read from your providers, and their yEnc headers must agree with the layout: decoded size, start offset, and total file size. A mismatch refuses the bundle and counts against the publisher. The cost is at most 24 article downloads (roughly 18 MB) per import. +- **Media structure.** Besides the start of each media file, its container index is read at several positions including the end: Matroska/WebM (EBML header, Segment size, and every SeekHead entry such as the Cues), MP4/MOV (the whole top-level box chain must end exactly at the end of the file, or, for fragmented files with thousands of boxes, a fragment box must end exactly there), MPEG-TS/M2TS (sync bytes on the 188, 192 or 204 byte grid at the start, quarters and end). Files muxed for streaming that have no Matroska SeekHead, or an unknown-size Segment, are accepted when a level-1 element opens the Segment, a known Segment size lands on the end of the file and the quarters and the end can be read; trailing free, skip or uuid boxes, a zero-sized last box, 64-bit box sizes and up to 64 KiB of zero padding in MP4 are accepted; other containers get full-length reads at the quarters and end. Every written media file passes this, not only the largest. +- **File types.** Only media files and small subtitle (`.srt .ass .sub .idx .sup`) and `.nfo` files are written (at most 40 media files and 50 sidecars, subtitles up to 16 MB, `.nfo` up to 1 MB). Files your *allowed file extensions* exclude are skipped like in a normal import. Playlists, links, queue files, scripts and executables (`.strm .nzb .nzbz .meta .url .lnk .m3u .m3u8 .pls .desktop .exe .sh ...`) make the whole bundle refused. +- **Folder name.** The release folder comes from the title with the media extension removed first; names that are empty, hidden (start with a dot), reserved by AltMount or equal to a category folder are refused, so a title can never point outside the category folder. +- **Offers.** Of several offers, the bundle announced identically by publishers in the most distinct network ranges is preferred; a publisher's own attestation never raises the rank of its own offer. + +What this proves: the articles exist on your providers, sampled articles match the layout, and the media container is structurally sound. It does not prove the whole file is the titled content; that is the same risk as a mislabelled NZB from an indexer. It does not prove the whole file is the titled content; that is the same risk as a mislabelled NZB from an indexer. + +### Stremio peer candidates + +When the peer network is enabled and a network is at level 1 or above, the Stremio stream list for a movie (or an episode) can include releases other nodes announced for that IMDb ID with a bundle you can fetch from a peer you have verified, and that you do not already have. They are marked "🌐 Peer network • verified on play", with "Healthy on the peer network" when enough nodes attest it, and come after your library streams. At most 10 are listed, healthy first, from one short lookup that is cached for a few seconds and gives nothing on any error. The lookup runs while the indexer search runs and never delays the list by more than a fraction of a second after the search finished. + +Playing one asks your node (not the player) to fetch the bundle, verify it as described above and write the files into the same folder a normal Stremio import of that category would use, then the player is redirected to the file; this typically takes a few seconds. The play link carries only the release ID. Only releases for which your node holds a signed record offering a fetchable bundle can be requested, the address dialled always comes from your verified peer table, concurrent plays of a release share one import, and plays are rate limited per key and overall (a few per minute). The IMDb ID, season and episode of the play request are kept for your own listing but marked unverified: they are never put into your own published records, because they come from the requesting player, not from anything proven. Stremio's play link can only be requested with your Stremio key. + +### Limits + +- Bundles: 8 MB compressed, 64 MB decompressed, 500,000 segments and 10,000 files at most; a receiving node also refuses releases over 2 TiB or 2,000 files. +- Fetching: on demand only, never speculative; at most 3 attempts of 60 seconds per request, 60 attempts per hour overall, 12 per publisher, 2 downloads at once. +- The health check must be enabled to receive bundles, because the full check must be able to run. +- A bundle is for the exact release: two NZBs of the same movie with different article sets are different releases. + ## Privacy At levels 2 and 3 the network can see which releases this node has, tied to its node ID and IP address. At level 3, article IDs are shared too. At level 1 only the bundles you ask for are revealed, to the peer you ask. Peers see each other's IP address; use the outbound proxy settings if that matters to you. diff --git a/docs/superpowers/specs/2026-11-12-peer-network-design.md b/docs/superpowers/specs/2026-11-12-peer-network-design.md index f1990a3d0..b0df0bda0 100644 --- a/docs/superpowers/specs/2026-11-12-peer-network-design.md +++ b/docs/superpowers/specs/2026-11-12-peer-network-design.md @@ -461,3 +461,519 @@ above. in batches; an exclusion entry named "unknown" matches no indexer, like the publisher; the fallback title drops a trailing media extension; the consumers' node-ID waiter makes a final check under its lock. + +## Phase 3: bundle format + +Implemented by `internal/peernet/bundle`: a pure package (no network, database +or filesystem). Schema version 1; an unknown major version is rejected. + +### Encoding + +Canonical protobuf wire format written by hand. Fields appear in ascending +number, repeated fields are consecutive, optional fields are omitted when +empty, varints are minimal. The decoder is strict (no unknown or duplicate +fields, no out-of-order fields) and finally re-encodes what it decoded: any +difference from the received bytes is a rejection, so a bundle has exactly one +byte representation. **Bundle ID** = SHA-256 of the canonical uncompressed +bytes. The wire form is zstd at its highest level (single frame, 8 MB window, +CRC). No dictionary: on synthetic data with realistic random message-IDs a +trained 84 KB dictionary saved 0.1 to 0.3 percent. Sizes, offsets and +references are run-length coded in the schema instead, so the layout costs a +few bytes per file and the message-IDs are essentially all of the payload. + +Typical size (random 20 to 31 character IDs, 716,800-byte segments, one file): +1 GB is about 1,500 segments, 64 KB canonical, 34 KB on the wire; 50 GB is about +75,000 segments, 3.2 MB canonical, 1.7 MB on the wire. The 8 MB wire cap is +reached at roughly 350,000 segments (about 2.5 TB of 716,800-byte articles), +below the 500,000 segment cap. + +### Schema (all integers are unsigned varints unless noted) + +| Message | Field | Content | +|---|---|---| +| Bundle | 1 version | must be 1 | +| | 2 scheme | 1 = nzb, 2 = files | +| | 3 release_id | 32 bytes | +| | 4 title | sanitised, at most 256 bytes | +| | 5 posted | Unix seconds of the release post | +| | 6 total_size | sum of the file sizes | +| | 7 password | optional opaque string from the NZB, at most 256 bytes | +| | 8 pool | repeated canonical message-IDs (no angle brackets) | +| | 9 release_files | nzb scheme only: packed counts, one per NZB file in NZB order | +| | 10 files | repeated File, sorted by File ID ascending | +| File | 1 name | sanitised base name | +| | 2 size | bytes, at least 1 | +| | 3 date | optional release date | +| | 4 segments | segment table (absent for nested files) | +| | 5, 6 aes_key, aes_iv | together, directly encrypted file | +| | 7 outer | repeated OuterSource | +| | 8 extents | packed triples (outer index, inner offset, inner length) | +| | 9 clips | packed pairs (byte length, zigzag 90 kHz delta) | +| | 10 par2 | repeated Par2File | +| | 11 file_id | 32 bytes, recomputed by Verify | +| OuterSource | 1 segments, 2 aes_key, 3 aes_iv, 4 volume_size | one outer archive volume; the key triple only when encrypted | +| Par2File | 1 name, 2 size, 3 segments | no gaps allowed | +| Segment table | 1 count | number of positions | +| | 2 refs | packed (base, count) runs; base is pool index + 1, 0 means gap | +| | 3 sizes | packed (decoded size, count) runs | +| | 4 partials | packed (index delta, start, end) for segments not used in full | + +A gap is a synthetic placeholder for an article the NZB never listed. It has no +message-ID in the bundle; the receiver derives a local placeholder id. + +### Identity: two ID schemes, both covered by the Bundle ID + +- **nzb**: Release ID is the existing Release ID over the full ordered segment + list. The pool holds that list in NZB order (including NZB files that were not + imported) and `release_files` keeps the per-file boundaries the hash needs. + Files refer to pool entries, so message-IDs are stored once. +- **files**: Release ID is `contentid.ReleaseIDFromFiles` over the bundle's File + IDs (domain `altmount-peernet/release-files/v1`, IDs sorted ascending + bytewise). The pool holds each message-ID once, in first-use order over the + sorted files (main segments, outer sources, PAR2). + +In both schemes every file's File ID is recomputed with the existing File ID +function from the segments the bundle describes (main segments plus each +extent's outer segments). `Verify(wire, bundleID, releaseID)` runs: size limits, +bounded decompression, hash equals the announced Bundle ID, strict decode and +structural validation, File IDs, then the Release ID for the declared scheme +must equal both the embedded and the announced one. A bundle presented with a +Release ID of the other scheme, or with a trimmed segment list, fails. + +### Limits (exported constants) + +8 MB compressed, 64 MB decompressed (enforced while streaming, 8 MB zstd +window, single-threaded decoder, no goroutines left behind), 500,000 segments +(pool, and positions over all tables), 10,000 files, 50,000 NZB file entries, +message-ID 248 bytes, file name 255 bytes, title 256 bytes, password 256 bytes, +10,000 outer sources per file, 100,000 extents per bundle, 100,000 clips per +file, 1,000 PAR2 files per file, segment 8 MiB, file and bundle 2^50 bytes, +nesting depth 1 (the schema cannot express a cycle or deeper nesting). + +### Reader assumptions and the checks that uphold them (summary) + +The full table with reader file and line references is in the comment at the top +of `validate.go`. + +- A segment's usable range is `[start, end]` with `0 <= start <= end < size`, + size at most 8 MiB (readers drop empty ranges and allocate `end + 1` bytes for + gaps). +- Placeholder and gap handling is explicit; a pool ID with the placeholder prefix + is refused. Message-IDs reach NNTP commands, so they are printable ASCII + without spaces or brackets. +- File size is at least 1. An unencrypted file's segments cover exactly its + size; an AES file's cover the padded size (at most 15 bytes more). +- AES keys are 16, 24 or 32 bytes and IVs 16 bytes, both present or both absent. +- Nested files have no inline segments or file-level key; extents lie inside the + encrypted volume size (or the plain outer coverage), do not overlap, are at + least 1 byte, and add up to the file size; every outer source is used; an + encrypted outer volume is covered by its padded size. +- Clip lengths are positive multiples of 192, add up to the file size, and + deltas are bounded. +- Names are single path components that are fixed points of the sanitiser, + unique ignoring case. Sums use checked arithmetic. + +### What is shared, dropped, or makes a release not shareable + +Shared: file name (sanitised base name), size, segments (message-ID, decoded +size, usable range), nested outer sources and extents, clip boundaries, PAR2 +file references, per-file release date, post date, title, optional NZB password, +and AES key and IV only when the publisher sets `PasswordFromNZB` for the file. + +Dropped: source and store paths, store reference, download ID, rclone password +and salt, known holes and the provider fingerprint, status, created and modified +times, indexer, category and ARR names, provider data. + +Not shareable (the builder returns a typed error for the whole release): any +rclone-encrypted file, any other encryption type, a file carrying key material +whose password is not from the NZB, a file healed by local PAR2 patches, a file +whose segments are not resolved or whose local metadata is inconsistent. + +### Conversion + +`FromLocal` builds a bundle from resolved per-file metadata; `ToLocal` returns +per-file metadata messages in the inline (v1) layout using shared outer sources. +Readers need no `.nzbz` store entry for such files. Names are sanitised base +names; the caller derives virtual paths locally. Placeholder ids and timestamps +are local. The nzb scheme also returns the per-file segment list so a later step +can compute the Release ID of the local import; it lacks subjects and groups, so +it cannot regenerate an NZB. + +## Library-derived release IDs and file-level evidence + +Added 2026-11-13. This section supersedes the "Release IDs need the whole +release" bullet of "Later changes (review round)": files imported in the older +on-disk metadata format (segments kept inline per file, no NZB store reference) +and files from stores rebuilt by the metadata migration are no longer left +without a Release ID. Found on a real library: 2,317 of 2,358 files had a File +ID and no Release ID, so they published nothing and received no network +evidence. + +### Decision + +Make older content shareable **without re-parsing NZBs**, at the cost of one +background read of metadata that the indexer already performs. The release's +full segment list is not available for such files, so the release is identified +by the set of its files instead. + +### The "files" Release ID (byte layout) + +``` +files release ID = SHA-256( u32be(len(L)) || L || fileID(1) || fileID(2) || ... || fileID(n) ) +L = "altmount-peernet/release-files/v1" (ASCII, 33 bytes, so the prefix is 00 00 00 21) +fileID(i) = the 32-byte File ID, unchanged; the list is the DISTINCT File IDs sorted ascending + by plain bytewise comparison; no count, no separators. +n >= 1 and n <= 10,000 (distinct File IDs). A zero File ID is invalid. +``` + +The domain label is framed exactly like the existing ones (u32 big-endian +length, then the label), and differs from the "nzb" Release ID label, so the two +ID spaces cannot collide. Version 1 must never change; a new layout needs a new +label. Implemented by `contentid.ReleaseIDFromFiles`; the bundle package +verifies the same rule by calling it. Duplicates are removed because one NZB +imported into two folders yields two rows with the same File ID, and a node +holding the duplicate must derive the same ID as one that does not (this +happens in the real library: 4 of 64 groups). + +### Which files make up "the release" (grouping rule) + +The files of one **import**, found through the metadata's `SourceNzbPath`: the +stored copy of the NZB the importer recorded in each file's metadata. It is +unique per import (it contains the queue item's own folder), so the key is a +hash of it (`group_key`, local only, never transmitted, never part of an ID). +Only the File IDs of the group reach the ID. + +Candidates and why they lost: + +- Release folder (parent directory): wrong both ways on the real library. A + folder can hold several imports (a season folder with one NZB per episode; + 2 of 78 folders did) and one import can span folders (nested release folders + and sample folders; 8 of 63 releases spanned more than one directory). A single + file sits directly in a category folder next to unrelated files, so a folder + rule would merge unrelated imports. +- `import_history` (nzb_id or folder-keyed virtual path): can be aged out. The + real library had 3 history rows for 2,358 files. +- `SourceNzbPath`: present on every file of the real library (2,358 of 2,358), + survives restarts and moves of the library (it is stored in the metadata, and + a folder rename does not change it), needs no extra read because the indexer + already reads the metadata to compute the File ID. + +A file with an empty `SourceNzbPath` cannot be grouped and gets no derived ID +(counted as `no_source_nzb`). A group with more than 2,000 files gets no ID (the ID function itself accepts up to 10,000, the bundle bound, so a bundle can be verified) +(`group_too_large`). A file with a store-based ("nzb") ID is never part of a +derived group and never gets a derived one; the derived group is the other files +of that import. A store the metadata migration rebuilt (`.nzbs/_migrated`) +still does not produce an "nzb" ID: its files now get a "files" ID. + +Stability: two nodes that imported the same NZB and kept the same file set +derive the same ID, whatever their paths or folder names. Nodes that kept +different subsets derive different Release IDs; File IDs match, so health +evidence is shared per file. This is accepted and documented for users. + +Known limitation: if a node's NZB storage reused one path for two different +NZBs, their files would be grouped together. The importer's per-queue-item +folder makes this unlikely; no case was found in the real library. + +### Storage + +Migration 057 (SQLite and PostgreSQL): + +- `peernet_content_index.release_scheme` (0 none, 1 nzb, 2 files; existing rows + with a `release_id` become 1) and `group_key` (local grouping key, partial + index on non-empty values). +- `peernet_records.release_scheme` (1 nzb, 2 files; default 1) and an index + `(network_id, file_id, kind)` for file-level lookups. + +A release ID with no scheme given to the repository is stored as scheme 1. + +### Pipeline + +- **Indexer.** `Entry` now also computes the group key from the metadata it + already reads. `IndexPath` writes only when the row's identity changed (file, + group, store-based ID), so metadata rewrites by health checks no longer touch + the index and never make a derived ID flicker. +- **Reconcile** already reads each row's metadata, so it fills in group keys on + rows written before the migration and treats a derived ID as part of the group, + not of the file. +- **Derive** (`DeriveAll`, `DeriveDirty`) reads only the index table. `DeriveAll` + is a keyset walk over groups (500 per page, at most 5,000 member rows per read, + one short transaction per write batch, cancellable between batches, resumable + and idempotent: a second pass over an unchanged index writes nothing). + Complexity: O(files) rows read, O(groups) queries, memory bounded by one + batch. Measured on a synthetic SQLite database of 200,000 rows in 40,000 + groups: first pass (every row written) 4.7 s, unchanged pass 1.8 s on a + developer laptop; a 500,000-row library extrapolates to about 12 s and 4.5 s. + It runs once per enable after backfill and reconcile. +- **Incremental.** The metadata observer marks the groups a write, delete or + rename touches; a debounced pass (3 s quiet, at most 30 s) derives only those. + Too many dirty groups, or a directory removal too large to track, falls back + to a full pass. Dirty state is in memory; a restart runs the full pass anyway. +- **Withdrawal.** When a group's derived ID changes or the group disappears (a + file added, removed, a file gaining a store, the last file deleted), the old + ID is withdrawn if no indexed file still carries it: this node's own records + for it are deleted in every network and its publish state forgotten, as + exclusions already do. Copies other nodes hold cannot be recalled and expire. + A rename of a folder changes no File ID and withdraws nothing. + +### Records + +Both record types carry an optional header field 9, `release_scheme`. Absent +means "nzb"; present must be exactly 2 ("files"); 0, 1 and any other value are +rejected as non-canonical. It sits between the common header (1-8) and the +type-specific fields (10+), is part of the signed body, and is covered by the +decode-then-re-encode identity rule. A record without it is byte-identical to the +encoding that predates it, so already-released records still verify and +re-encode identically. Nodes that predate the field reject a record that carries +it (unknown field); that is the safe direction. The store keeps the scheme +beside each record and `CandidatesByMedia` returns it. + +### Health evidence by file + +An old-format import and a fresh import of the same NZB have different Release +IDs and identical File IDs, so the consumer matches attestations by +`(network, file_id)` whatever Release ID they were published under: + +- `AttestationsForFiles(network, fileIDs, notBefore)`, bounded to 1,024 files, + returns at most one row per (file, publisher, verdict) and at most 5,000 rows, + unhealthy first, so truncation can only drop healthy evidence. The index + `(network_id, file_id, kind)` serves it (the query defeats SQLite's habit of + preferring the `(network, expires_at)` index when it has no statistics). +- `HealthAdvice` and the recheck pass use it. The release ID is optional in + `HealthAdvice`, so a file with only a File ID now gets advice. Every rule is + unchanged: freshness, coverage, backbone subset, own records excluded, known + publishers, distinct network prefixes, the private-network and open-network + thresholds, conservative mixed evidence. Recheck evidence is paid for once per + file (the budget key no longer includes the release ID), so the same attestation + reached through two release IDs cannot be paid for twice. +- **Label ("healthy on the network").** A release is labelled when **every one + of its video files** (known extensions, at most 16 per release) has qualifying + evidence under the existing per-release rule applied per file (enough + eligible publishers, distinct prefixes on the open network, no unhealthy + evidence at the threshold). Reasoning: a release is as available as its worst + file, and a publisher that attested only some of its files has not vouched for + the release; this is stricter than the previous per-release tally, where one + attested file counted for the whole release. A release with no video file, or + more than 16, is not labelled. One lookup per receiving network serves a whole + page (at most 200 releases and 1,024 files), with one batched publisher-standing + lookup per network. + +Security reasoning. File IDs are derivable by anyone holding the NZB, and so +were Release IDs, so file-level matching hands an attacker nothing they could +not already do: they could already attest any release/file pair. What changed and +how it is handled: + +- An attacker no longer needs the right Release ID; the File ID is the + same knowledge. The claim ("these articles are healthy") is about the file. +- One publisher could in principle fill a file's lookup with records under many + invented Release IDs (5,000 per publisher are allowed). The query keeps one + row per (file, publisher, verdict), so a publisher counts once per file and + cannot crowd out others. +- Corroboration still needs distinct known publishers in distinct network + prefixes; an own or relayed-own record never counts; nothing here writes a + status. The label became stricter, not looser. + +### Publishing and status + +Old-format files publish like any other: release record and attestation, with +`release_scheme` 2. All exclusion, password, title and skip rules are unchanged. +The title comes from the same sources (history name, queue name, the metadata's +source NZB name with a media extension stripped); on the real library no derived +group had an empty title. Total size and file count come from the grouped files +(distinct File IDs). A derived release of up to 2,000 library files is described; +larger ones get an ID but publish nothing (`no_release_info`). + +`GET /peernet/status` `library` now reports `with_nzb_release_id`, +`with_derived_release_id` and the reasons `no_source_nzb`, `group_too_large`, +`pending` next to the existing fields; the Activity card says so in plain +language. + +### Real-library result (read-only check on a copy) + +2,358 indexed files, 41 with an "nzb" ID. After the pass: 41 "nzb", 2,317 +"files", 0 unshared. 64 derived groups (sizes: 36 single files, 10 pairs, 2 of +3-5, 4 of 6-10, 8 of 11-50, 2 of 51-500, one of 1,570); the largest five are +1,570, 382, 148, 39 and 23 files. 5 groups span more than one release folder: +four are one import kept under several folders (identical File IDs, now counted +once) and one is a two-file NZB whose files sit in two folders. None merges +unrelated imports. A folder rule would have merged unrelated imports in 2 of 78 +folders. + +## Phase 3: building, serving and fetching bundles + +Implemented by `internal/peernet/bundlex` and the service. Level 3 only serves. + +- **Build.** At publish time, when a release record is about to be signed for a + network at level 3, the bundle is built from local data (one build at a time, + off the health worker's goroutine) so the signed record carries the Bundle ID. + The builder reads full per-file metadata only through its own interface + (`bundlex.BuildSource`), which nothing else in the peernet packages receives. + Not shareable, counted by reason: rclone encryption, other encryption, key + material whose password is not proven to be inside the NZB, a PAR2 patch + over any article (or patch status unknown), known holes, no NZB segment list + for an "nzb" release (it never falls back to "files"), a rebuilt Release ID + that differs from the indexed one, limits exceeded. Such a release still + publishes its record, without a Bundle ID. +- **Password origin.** The metadata keeps an archive's AES key and IV but not the + password or where it came from (the importer holds it for the import only and + the stored NZB has no meta head). The origin is therefore never known, so key + material is never shared and encrypted-archive releases are not shareable until + the importer records the origin. +- **No stored bundles.** A rebuild of a 50 GB, 75,000-segment release takes about + 0.11 s of CPU (including a self-check with the receiver's verifier) plus the + metadata read, and builds are deterministic, so only the Bundle ID is stored + (publish state) and the last bundles stay in a 32 MB memory cache for 10 + minutes. A rebuild whose ID differs from the announced one is never served; the + record is marked to be announced again. +- **Level changes.** The publish state remembers the level a record was published + at and its Bundle ID. A network moving to level 3 (or below it) supersedes its + records at the next upkeep pass. +- **Serve.** `GET /peer/v1/bundle?n=&b=` on the peer listener. + Level below 3, unknown network, denied caller, a Bundle ID this node did not + announce in that network, or a release excluded since all get the same + undecorated 403. Per-peer 30 per hour (burst 5), per public address 60 per hour + (burst 10; private ranges are not limited per address), 128 MB per minute + node-wide (burst 32 MB), 4 at once (1 per peer, 2 per address); no ranges, no + compression; peers record exchange has backed off are refused. +- **Fetch.** On demand only. Offers are stored release records with a Bundle ID, + ordered known publisher, healthy attestation, newest; only publishers with a + verified address are dialled. At most 3 attempts, 60 s each, 8 MB cap while + streaming, `bundle.Verify` against the IDs and scheme of the signed record. + Failed verification or an oversized body backs the publisher off (10 minutes, + doubling, capped at 2 hours) for bundle fetches only; record sync is unaffected. + 60 attempts per hour node-wide, 12 per publisher, 2 downloads at once. + +## Phase 3: wiring decisions + +Added when the publishing and the receiving halves of phase 3 were connected. + +- **Fetcher and reporter.** The service is the importer's bundle fetcher (the + fetch source becomes the importer's origin) and its bad-bundle reporter. The + importer's fast path is installed only while the feature is enabled and a + fetcher is connected, and acts only while a network is at level 1 or above + (checked on every import). With the feature off none of the hooks exists. +- **Bad-bundle back-off (fetch domain only).** A bundle that passed hash + verification but whose release then failed the local full check backs its + publisher off for bundle fetches: 30 minutes, doubling per report, capped at 6 + hours, per (network, publisher), with the doubling forgotten after 24 hours + without a report, cleared at once by a later release from that publisher whose + every file passed its full check, bounded in memory like the other fetch + tables, and counted (`full_check_reports`, `full_check_backed_off`). It is kept + apart from the hostile-bytes strikes and never touches records, record sync or + stored data. Why mild: a full-check failure is also what a receiver whose own + providers lack articles sees, so the same honest publisher must not be shut + out for long, while repeated failures still escalate. Abuse by a publisher is + already bounded by hash and identity verification: a publisher cannot make a + bundle that passes those and also reliably fails a full check on every + provider. +- **Publish eligibility.** A file that came from a peer bundle is not published + (record, attestation, bundle, re-announcement) until the importer's persistent + state says its full check passed. The publishing pipeline asks a narrow + interface (`PeerBundleStatus`) for each healthy notice and for every file of a + release before a re-announcement; an error means not eligible (skip reason + `peer_status_unknown`), a pending file is skipped with + `peer_bundle_not_fully_checked` through the publisher's existing + `FromPeerBundle`/`FullCheckPassed` inputs (which nothing set before). The truth + is the persistent state, not the size of the one check that triggered the + notice, so later sampled re-checks of a verified file publish like any local + import. The health worker now tells the gate before it tells the publishing + observer, so a notice never finds the pass unrecorded. +- **Patch index.** The PAR2 patch store is handed to the builder, so bundle + sharing is not blocked as `patch_status_unknown`. +- **Release identity of bundle-imported files.** Their metadata has no source + NZB path and no store, so nothing in it can reproduce the Release ID the + receiver verified. Accept records the verified Release ID and scheme directly + in the content index (before the metadata is written, so the observer finds + the row) with a local group key derived from that ID. No schema change: a row + whose group key is the key of its own Release ID is a verified bundle row, and + a row with a local group key whose metadata names no source NZB and no store + is never replaced by an entry computed from the metadata (index, reconcile, + backfill). The derive pass treats such rows like any group: for the "files" + scheme an intact set derives the same ID (nothing is written) and a deleted + file changes the group's ID and withdraws the old one exactly as for a local + import; an "nzb" scheme ID names the NZB and survives deleting a file. + Consequence: a bundle-imported "nzb" release has no NZB store, so it cannot be + rebuilt as a bundle (`no_nzb_list`); its record and attestations can be + published. +- **Password origin.** The decision: record it for new imports, never share keys + of unknown origin. + - *Where it is stored.* A side table `peernet_password_origin` (migration 061) + keyed by File ID, one row per file whose archive password came with the NZB's + own content; no row means "not from the NZB", which is what every earlier + import is. A File ID key survives renames and moves (the path changes, the + segments do not) and does not depend on the asynchronous content indexer. The + proto was not touched: the committed generated file does not match what the + Makefile's protoc invocation produces, so regenerating risks changing symbol + names for a one-bit fact. + - *Which origins set it.* Only a `` (the parsed meta head, + or a meta element elsewhere in the NZB's first 64 KB) that is the password in + effect, in an NZB the API did not alter, and not a nested NZB read from inside + another release's archive. Not set: a password spelled in the NZB file name + (`{{pw}}`), in a file subject, typed by the user (`&p=` of a link or the + password field of a form, which the API embeds as a meta element before + enqueueing), supplied by an ARR through the SABnzbd API (only the name form + exists there; an NZB the ARR delivers with its own meta counts, as it is NZB + content), or found for a file with no key material. The API marks an + injected password in the queue item's metadata (`_altmount_pw_injected`, never + the password); because the parser lets the NZB's own meta win over an embedded + one, an injected password on an NZB that also carries its own meta password is + conservatively treated as injected. + - *Written when.* After a successful import, for the written files that carry + key material, by a hook the importer has only while the peer network is + enabled (so imports made while it is off carry no flag). + - *What the builder answers.* "From the NZB" with no password text, because the + password is never stored: the bundle carries the key and IV, which is all a + receiver needs to play it. The password is only used if the user later + reprocesses the release. + - Bundle-imported files carry no flag: their key's origin is the publisher's + claim, which a receiver cannot check, so they are not re-shared with keys. +- **Stremio flow.** `GET /stremio/:key/stream/...` appends peer candidates after + the library streams: records for the IMDb ID (and season and episode) from the + receiving networks, not own, not already in the content index, with a fetchable + bundle offer (publisher is a verified peer with an address), healthy-labelled + first, at most 10, one bounded lookup (3 s timeout, 20 offer checks, a 15 s + cache), failing open. Titles are length-capped and stripped of control + characters and only ever sit inside JSON text; the marker is a fixed local + string. The stream URL is `GET /stremio/:key/peer/:release` (64 lower-case hex + release ID, content type and IMDb ID in the query), in the same key-checked + group. It validates the ID, rate limits (per key 6 per minute, burst 3, and 20 + per minute overall, burst 6), requires a stored signed record offering a + fetchable bundle (a local read, no fetch otherwise), runs the import once for + concurrent requests (detached from any one client, 2 minute limit), writing into + the folder a normal import of the category (Movies or TV) would use + (`PeerImportDir`), with the queue metadata and a `stremio:` download ID a + Stremio-originated import carries (IMDb ID, type, season and episode; no + TMDB/TVDB lookup, which would be an external call per play), so this node's own + republished record carries the media IDs and the Stremio cleanup treats the + import like any other. It then redirects to the library stream of the main + media file (largest non-sample file; for an episode, the matching one). Any + failure answers 503 with a fixed message. No configuration switch was added: + the feature follows the peer network being enabled with a receiving network and + a connected fetcher; a `peernet.stremio_candidates` key would need config + patching and UI work for little gain. +- **Title of a bundle import** is the bundle's sanitised title. + +## Phase 3: hardening after review + +A security and a correctness review of the receiving half of phase 3 produced these rules. A bundle is a peer's description of how to read a release; every byte of it is hostile. + +- **No gaps in bundles.** A shareable file has no known holes. `Validate` rejects any gap reference (files, outer sources, PAR2); the builder refuses a placeholder segment (`not shareable`). The wire format can still spell a gap, but no decoder result containing one validates. +- **Coverage rule (nzb scheme).** The pool is the NZB's segment list in NZB order, ids unique. Every segment table (inline file, outer volume, PAR2 file) must be exactly one NZB file entry's segments, in order. An entry read inline belongs to one file and is read once; an outer volume or PAR2 file may be shared by several files (a multi-file archive) but only once per file and never also inline. +- **Decode budgets.** Clips (200,000), outer sources (50,000) and PAR2 files (10,000) have bundle-wide budgets next to segments and extents; packed fields are counted before allocation and each file is validated as soon as it is decoded. A hostile 64 KB input is bounded by a test (allocations under 128 MB). +- **Sizes against the user's NZB.** `` is the encoded size (a few percent above decoded). A declared decoded size must be at most the encoded one and not below 90 percent of it minus 512 bytes per segment, per segment and per table. +- **Sampled yEnc verification.** Before writing, on every accept path, the first and last article of each table and two random ones (crypto/rand, at most 24 in all) are fetched and their yEnc headers (part size, 0-based begin offset, total) must equal the layout (declared size, cumulative offset, sum of the table). The pool cannot stop after the headers, so a sample costs a whole article. A mismatch is attributable to the publisher and is reported (fetch-domain back-off). No header reader or a missing article rejects the bundle. +- **Structural media verification.** Every written media file passes the head signature check and then a container check through the staged layout: Matroska/WebM EBML header, Segment size and every SeekHead target (this reaches the Cues at the tail); MP4/MOV top-level box chain ending exactly at the file end with moov and mdat; MPEG-TS/M2TS sync at the start, quarters and the last packet; other containers get full-length reads at quarters and the tail. There is no deeper media parser in the repository. +- **Fast-path policy.** The importer's fast path takes private-network offers without a switch. Open-network offers need `peernet.open_network.fast_path` (default false) and at least two corroborating publishers (distinct /24 or /48 ranges announcing the identical Bundle ID). Fetch (5 s) and accept (15 s) are separately bounded; a layout that cannot be verified in time is declined and the NZB imports normally. Direct imports (Stremio peer play) do not use the switch. +- **Corroborated offers.** `Offer.Corroborators` counts distinct network ranges of known verified publishers announcing the same Bundle ID. Order: known, attested healthy by someone else, corroborators, newest. A publisher's own attestation never counts for its own offer. +- **Extension policy.** The importer's allowed-extensions list is applied (disallowed files are skipped, as the importer does). A fixed forbidden list (`.strm .nzb .nzbz .meta .url .lnk .m3u .m3u8 .pls .desktop`, executables and scripts) refuses the whole bundle. Written files are verified media (at most 40) plus subtitles and `.nfo` under size caps (at most 50); everything else is not written. +- **Folder naming.** `filesystem.SafeFolderName` strips the media extension first, rejects empty, dot-leading, reserved (`.nzbs corrupted_metadata patches _migrated .peernet-staging`) and category-equal names, caps length, and the importer's `PeerImportDir` joins it directly (never through `NzbFolderPath`). `cleanTarget` requires at least two components and no reserved component. +- **Write order.** Pending state rows, then metadata files, then the content index, under a per-target lock, with an undo that runs on error, panic or cancellation and removes only what the call created. A second accept of the same release waits and finds it already imported. A file with the same File ID at the path counts as the same import only when its recorded Release ID matches. Names are compared NFC and case-insensitively. +- **Pending lifetime.** A file for which the health system will never create a record (`postprocessor.WillScheduleHealth`) is marked passed at once. Pending files older than 7 days are failed. The health gate stays installed while any file is pending even when the feature is off. Delete, rename and folder rename keep the state consistent through a secondary metadata observer; a rename keeps the pending state. +- **Media IDs of peer plays.** The request's IMDb ID, type, season and episode are stored under `unverified_*` keys in the queue metadata; the publisher's media-ID reader ignores them. +- **Serve side.** Serve builds have their own slot and a node-wide rate (3 burst, 360 per hour), remember an unbuildable release for a minute, and never wait behind or block the publishing builds. An item larger than the cache cap is not cached. Full strike tables evict the entry that expires first. +- **ARR visibility while verifying.** The hold is its own persisted mark, `import_queue.peer_hold` (migration 062), not the user's `skip_arr_notification`, which keeps meaning only what the user asked. A held item is a completed queue row. The SABnzbd-compatible queue lists held items (the active list widened by `peer_hold <> 0`, no extra query) as `Downloading`, 99 percent, a status this API already emits for work in progress and every ARR client treats as active, never as terminal; the history excludes them (`completed` arm: `peer_hold = 0`; by-id lookups skip them). A pass clears the mark and the item reaches the history as completed through the normal path, the ARR notification following. A failure (full check failed, or the 7-day stale escalation) sets the item `failed` with a short message and removes its provisional history rows, so the history shows it as failed and the ARR blocklists and searches again. A user's skip keeps the item out of queue and history in every state. An import no ARR is told about (caller-suppressed or direct) ends its hold when the last file passes. A queue item the user removed is not announced. +- **Failure path per origin.** One path for every origin: `FullCheckFailed` marks the file failed, then `failImport` fails the held queue item, removes every file of the import (metadata through `RemoveFailedFile`, which is the health worker's delete-on-corruption removal plus the health record, content index, state rows) and never touches anything that is not a row of that import. The health worker stops before any ARR repair for a corrupted verdict on a pending peer import (`IsPendingPeerImport`, fail-open) and lets the gate end the import after the verdict is written, so the same item is never acted on twice. An ARR-originated item: the ARR learns from the failed history entry. A direct import (Stremio peer play): there is no ARR and no special Stremio failure behaviour exists in the health path (`stremio.skip_repair` only governs PAR2 repair during import), so it follows the same removal; the queue row shows failed. Stale: identical to a failed check except that the publisher is not reported. +- **Terminal-safe queue rows.** The queue row of a direct import is created `completed` with a marker path (`peernet-bundle/...`, `IsPeerImport`), held at once and rolled back when a later step fails, so a restart never retries it into a failure; retry, bulk restart and NZB download of such a row are refused with a clear message (409). Deleting the queue item of a pending import is allowed; the state rows keep verifying and announce nothing. +- **Startup reconcile.** Rows are written uncommitted (`peernet_bundle_imports.committed`, migration 062) before the first file and committed as the last step of `accept.write`; an uncommitted row blocks announcement and publishing. At start `Reconcile` rolls every uncommitted import back whole (files whose ID matches the row, index entries, rows), drops rows whose files no longer exist and lifts holds nothing refers to. Tests abandon `write` after every step (`rows`, each file, `files`, `index`). +- **Stremio listing.** Peer candidates are looked up concurrently with the indexer search and collected afterwards with at most a 250 ms grace; a slow, failing or panicking lookup yields no peer streams. The merge (library, peer, indexer entries) is one function and byte-identical to the former output when there are no peer streams. +- **Cheap no-offer import.** The importer asks `MayHaveOffers` first: a cached answer, recomputed only when a bundle-announcing record was stored (`BundleOfferGeneration`), so with no offer nothing is built or hashed. When offers exist the store built for the lookup is reused by the import's parse unless gap placeholders changed the NZB. +- **Structure check variants.** See the matrix in the user documentation: SeekHead-less and unknown-size Matroska, fragmented MP4, trailing boxes and zero padding, 64-bit and zero-sized boxes, and 188, 192 and 204 byte transport streams are accepted; a layout correct only at the head is still rejected, because the end of the file is always read (SeekHead target, box chain, fragment box ending at EOF, sync on the last packet). The weakest case is a SeekHead-less Matroska, where the end is checked by the Segment size and readable tail only. diff --git a/frontend/src/components/config/peernet/ActivityCard.tsx b/frontend/src/components/config/peernet/ActivityCard.tsx index cbda5dd99..7f9362154 100644 --- a/frontend/src/components/config/peernet/ActivityCard.tsx +++ b/frontend/src/components/config/peernet/ActivityCard.tsx @@ -1,10 +1,17 @@ +import { formatBytes } from "../../../lib/utils"; import type { PeernetStatus } from "../../../types/peernet"; import { backboneHint, describeBreakdown, + fetchFailLabel, + importRejectLabel, libraryShareLine, + notShareableLabel, publishedTotal, + serveRefusedLabel, showAdviceBlock, + showBundleServing, + showBundlesBlock, showPublishingDetails, skipLabel, } from "./peernetStats"; @@ -58,6 +65,78 @@ function PublishingBlock({ status }: { status: PeernetStatus }) { ); } +interface BreakdownListProps { + title: string; + counts: Readonly> | undefined; + label: (key: string) => string; +} + +function BreakdownList({ title, counts, label }: BreakdownListProps) { + const entries = describeBreakdown(counts, label); + if (entries.length === 0) return null; + const total = entries.reduce((sum, e) => sum + e.count, 0); + return ( +
+

+ {title}: {total} +

+
    + {entries.map((entry) => ( +
  • + {entry.label}: {entry.count} +
    • + ))} +
+
+ ); +} + +function BundlesBlock({ status }: { status: PeernetStatus }) { + const b = status.bundles; + const imp = b?.import; + const serving = showBundleServing(status.networks); + const backedOff = count(b?.fetch_backed_off); + return ( +
+ {serving && ( +
+

Bundles announced: {count(b?.announced)}

+

+ Bundles served: {count(b?.served)} ({formatBytes(count(b?.served_bytes))}) +

+ + +
+ )} +
+

Bundles fetched: {count(b?.fetched)}

+

Imports accepted: {count(imp?.accepted)}

+

+ Awaiting full check: {count(imp?.full_check_passed)} passed,{" "} + {count(imp?.full_check_failed)} failed +

+ {(imp?.pending ?? 0) > 0 && ( +

+ Waiting for a full check: {count(imp?.pending)} + {(imp?.stale ?? 0) > 0 ? ` (${count(imp?.stale)} waited too long and were failed)` : ""} +

+ )} + + + {backedOff > 0 &&

Publishers backed off: {backedOff}

} +
+
+ ); +} + function AdviceBlock({ status }: { status: PeernetStatus }) { const advice = status.advice; return ( @@ -97,6 +176,14 @@ export function ActivityCard({ status }: ActivityCardProps) { + {showBundlesBlock(status.networks) && ( +
+

+ Bundles +

+ +
+ )} {showAdviceBlock(status.networks) && (

diff --git a/frontend/src/components/config/peernet/ExposureLevelSelect.tsx b/frontend/src/components/config/peernet/ExposureLevelSelect.tsx index fc0192aa0..5b905389f 100644 --- a/frontend/src/components/config/peernet/ExposureLevelSelect.tsx +++ b/frontend/src/components/config/peernet/ExposureLevelSelect.tsx @@ -1,8 +1,53 @@ import { ShieldAlert } from "lucide-react"; -import { useId } from "react"; +import { useEffect, useId, useRef, useState } from "react"; import type { PeernetLevel } from "../../../types/peernet"; -import { isLevelAvailable } from "./peernetStats"; -import { EXPOSURE_LEVELS, levelNeedsPrivacyNotice, levelPrivacyNotice } from "./peernetValidation"; +import { levelNeedsConfirmation } from "./peernetStats"; +import { + EXPOSURE_LEVELS, + LEVEL3_PRIVACY_NOTICE, + levelNeedsPrivacyNotice, + levelPrivacyNotice, +} from "./peernetValidation"; + +interface ShareBundlesConfirmProps { + open: boolean; + onConfirm: () => void; + onCancel: () => void; +} + +function ShareBundlesConfirm({ open, onConfirm, onCancel }: ShareBundlesConfirmProps) { + const ref = useRef(null); + const titleId = useId(); + useEffect(() => { + const dialog = ref.current; + if (!dialog) return; + if (open && !dialog.open) dialog.showModal(); + if (!open && dialog.open) dialog.close(); + }, [open]); + return ( + +
+

+ Share bundles? +

+

{LEVEL3_PRIVACY_NOTICE}

+
+ + +
+
+
+ +
+ + ); +} interface ExposureLevelSelectProps { name: string; @@ -18,6 +63,11 @@ export function ExposureLevelSelect({ disabled = false, }: ExposureLevelSelectProps) { const uid = useId(); + const [confirming, setConfirming] = useState(false); + const select = (level: PeernetLevel) => { + if (levelNeedsConfirmation(value, level)) setConfirming(true); + else onChange(level); + }; return (
Exposure level @@ -26,14 +76,11 @@ export function ExposureLevelSelect({ const id = `${uid}-${name}-level-${info.level}`; const titleId = `${id}-title`; const descId = `${id}-desc`; - const unavailable = !isLevelAvailable(info.level); return (
); } diff --git a/frontend/src/components/config/peernet/OpenNetworkCard.tsx b/frontend/src/components/config/peernet/OpenNetworkCard.tsx index f75260bfb..38d20a108 100644 --- a/frontend/src/components/config/peernet/OpenNetworkCard.tsx +++ b/frontend/src/components/config/peernet/OpenNetworkCard.tsx @@ -50,6 +50,24 @@ export function OpenNetworkCard({ Private, loopback and similar local addresses are not connected to on the open network.

+
+ Use offered layouts when importing + +
+
Minimum corroboration = new Map([ ["not_healthy", "Not healthy locally"], ["reduced_by_peer_advice", "Check was shortened by network evidence"], ["patch_status_unknown", "Repair status of the file was not known"], + ["peer_status_unknown", "Could not tell whether the file came from a peer bundle"], + ["bundle_unavailable", "Bundle could not be built for this file"], ]); +const NOT_SHAREABLE_LABELS: ReadonlyMap = new Map([ + ["rclone_encrypted", "Encrypted with rclone"], + ["unsupported_encryption", "Uses an encryption type that cannot be shared"], + ["password_not_from_nzb", "Password did not come inside the NZB"], + ["healed_by_local_patch", "Repaired with local PAR2 patches"], + ["known_holes", "Has known missing parts"], + ["patch_status_unknown", "Repair status was not known"], + ["no_nzb_segment_list", "No stored NZB segment list"], + ["release_id_mismatch", "Release identity did not match"], + ["invalid_metadata", "Stored file details are inconsistent"], + ["too_large", "Too large to bundle"], + ["no_post_date", "No post date known"], + ["no_files", "No files in the release"], + ["self_check_failed", "The built bundle failed its own check"], + ["other", "Other reasons"], +]); + +const SERVE_REFUSED_LABELS: ReadonlyMap = new Map([ + ["unauthenticated", "Caller not authenticated"], + ["level", "Network level does not serve bundles"], + ["denied", "Caller not allowed"], + ["backed_off", "Caller temporarily blocked"], + ["rate_limited", "Caller over its request rate"], + ["busy", "Too many requests at once"], + ["error", "Internal error"], + ["unknown_bundle", "Bundle not known"], + ["excluded", "Release is excluded from sharing"], + ["unavailable", "Bundle could not be built"], + ["byte_budget", "Node-wide data limit reached"], + ["aborted", "Transfer was cut off"], +]); + +const FETCH_FAILED_LABELS: ReadonlyMap = new Map([ + ["refused", "Peer refused the request"], + ["bad_status", "Peer answered with an error"], + ["transport", "Connection problem"], + ["truncated", "Download was cut short"], + ["too_large", "Bundle larger than allowed"], + ["verification_failed", "Bundle did not match its signed record"], + ["scheme_mismatch", "Bundle format did not match"], + ["membership", "Peer not a verified member"], +]); + +const IMPORT_REJECT_LABELS: ReadonlyMap = new Map([ + ["invalid", "Failed validation"], + ["bad_name", "Unsafe file name"], + ["too_large", "Over the local size limit"], + ["too_many_files", "Over the local file limit"], + ["health_disabled", "Health checks are turned off"], + ["bad_target", "Target folder unusable"], + ["nzb_mismatch", "Did not match your NZB"], + ["collision", "A different file is already there"], + ["probe_missing", "Articles missing on your providers"], + ["probe_inconclusive", "Article check could not finish"], + ["no_media_file", "No media file to verify"], + ["media_invalid", "Start of the media file was not valid"], + ["media_missing", "Start of the media file is missing"], + ["media_inconclusive", "Media check could not finish"], + ["staging_failed", "Could not prepare files"], + ["write_failed", "Could not write files"], + ["unavailable", "No bundle could be fetched"], + ["no_network", "Peer network not available"], + ["forbidden_file", "Contains a file type that is never accepted"], + ["sidecar_too_large", "A subtitle or info file was too large"], + ["size_implausible", "Declared file sizes do not fit your NZB"], + ["yenc_mismatch", "Sampled articles contradict the bundle"], + ["structure_invalid", "Media file structure is broken"], +]); + +export function notShareableLabel(key: string): string { + return NOT_SHAREABLE_LABELS.get(key) ?? key; +} +export function serveRefusedLabel(key: string): string { + return SERVE_REFUSED_LABELS.get(key) ?? key; +} +export function fetchFailLabel(key: string): string { + return FETCH_FAILED_LABELS.get(key) ?? key; +} +export function importRejectLabel(key: string): string { + return IMPORT_REJECT_LABELS.get(key) ?? key; +} + export function rejectionLabel(key: string): string { return REJECTION_LABELS.get(key) ?? key; } @@ -125,21 +209,106 @@ export function showAdviceBlock(networks: readonly HasLevel[]): boolean { return networks.some((n) => n.level >= 1); } -/** Level 3 (bundles) is not implemented; it behaves like level 2 for records. */ -export function isLevelAvailable(level: PeernetLevel): boolean { - return level !== 3; +/** The bundles block needs a network that receives. */ +export function showBundlesBlock(networks: readonly HasLevel[]): boolean { + return networks.some((n) => n.level >= 1); +} + +/** Only level 3 serves bundles. */ +export function showBundleServing(networks: readonly HasLevel[]): boolean { + return networks.some((n) => n.level >= 3); +} + +function positive(n: unknown): boolean { + return typeof n === "number" && Number.isFinite(n) && n > 0; +} + +function anyPositive(value: unknown): boolean { + if (positive(value)) return true; + if (value && typeof value === "object") return Object.values(value).some(anyPositive); + return false; +} + +/** True when any counter in the bundles object is non-zero. */ +export function bundleHasActivity(bundles: unknown): boolean { + return anyPositive(bundles); +} + +/** Raising a network to level 3 (bundle serving) needs an explicit confirmation. */ +export function levelNeedsConfirmation(from: number, to: number): boolean { + return to === 3 && from !== 3; +} + +export const PEER_BADGE_LABEL = "Peer network"; +export const PEER_BADGE_VERIFYING_LABEL = "Peer network \u00b7 verifying"; +export const PEER_BADGE_VERIFYING_TITLE = + "Imported from a peer bundle. It is playable now; a full check of every segment is running. It is announced to your ARRs and shared only after that check passes."; +const PEER_BADGE_TITLE = "Imported from a peer bundle."; + +export interface PeerBadgeInfo { + label: string; + title: string; +} + +/** Badge text for an item imported from a peer bundle, or null for any other item. */ +export function peerBadge( + origin: boolean | undefined, + fullCheckPending: boolean | undefined, +): PeerBadgeInfo | null { + if (origin !== true) return null; + return fullCheckPending === true + ? { label: PEER_BADGE_VERIFYING_LABEL, title: PEER_BADGE_VERIFYING_TITLE } + : { label: PEER_BADGE_LABEL, title: PEER_BADGE_TITLE }; +} + +function count(n: unknown): number { + return typeof n === "number" && Number.isFinite(n) && n > 0 ? n : 0; +} + +function countedPhrase(n: number, singular: string, plural: string): string | null { + return n > 0 ? `${n} ${n === 1 ? singular : plural}` : null; } /** * One plain-language line on how much of the library can be shared, or null * when there is nothing to say (no files known yet, or all can be shared). + * Older imports (no stored NZB) are identified by the files they contain, so + * they count as shareable; the line names why the remaining files are not. */ export function libraryShareLine(library: PeernetLibraryStatus | undefined): string | null { if (!library) return null; const { files, with_release_id: shareable } = library; if (![files, shareable].every((n) => Number.isFinite(n) && n >= 0) || files <= 0) return null; if (shareable >= files) return null; - return `${shareable} of ${files} library files can be shared; the rest were imported before sharing data was kept.`; + + const older = count(library.with_derived_release_id); + const shared = + older > 0 + ? `${shareable} of ${files} library files can be shared (${older} of them ${older === 1 ? "is an older import" : "are older imports"}, identified by the files they contain).` + : `${shareable} of ${files} library files can be shared.`; + + const reasons = [ + countedPhrase( + count(library.no_source_nzb), + "has no record of the import it came from", + "have no record of the import they came from", + ), + countedPhrase( + count(library.group_too_large), + "is in an import with too many files", + "are in imports with too many files", + ), + countedPhrase( + count(library.pending), + "is still being worked out", + "are still being worked out", + ), + countedPhrase(count(library.not_indexable), "could not be read", "could not be read"), + ].filter((r): r is string => r !== null); + const rest = files - shareable; + return reasons.length > 0 + ? `${shared} The other ${rest} cannot be: ${reasons.join(", ")}.` + : `${shared} The other ${rest} cannot be.`; } /** The skip key that means no provider backbone could be recognised. */ diff --git a/frontend/src/components/config/peernet/peernetValidation.ts b/frontend/src/components/config/peernet/peernetValidation.ts index 336580c05..856070648 100644 --- a/frontend/src/components/config/peernet/peernetValidation.ts +++ b/frontend/src/components/config/peernet/peernetValidation.ts @@ -20,7 +20,7 @@ export const DEFAULT_PEERNET_CONFIG: PeernetConfig = { accept_incoming: false, listen_port: DEFAULT_LISTEN_PORT, advertise_address: "", - open_network: { level: 0, peers: [] }, + open_network: { level: 0, peers: [], fast_path: false }, sharing: { excluded_categories: [], excluded_indexers: [] }, min_corroboration: 3, }; @@ -37,7 +37,7 @@ export const EXPOSURE_LEVELS: readonly ExposureLevelInfo[] = [ level: 1, name: "Receive only", description: - "Receives release records and health information from peers and shares none of your releases. Peers can only see that this node is connected, through its node ID and IP address.", + "Receives release records and health information from peers and shares none of your releases. Peers can only see that this node is connected, through its node ID and IP address. It can also receive a bundle on demand, when you play or import a release a peer offers; that reveals the request to the peer asked.", }, { level: 2, @@ -49,17 +49,18 @@ export const EXPOSURE_LEVELS: readonly ExposureLevelInfo[] = [ level: 3, name: "Share bundles", description: - "Not available yet. Serving finished imports to peers is not implemented, so this level cannot be selected. A network already set to 3 behaves exactly like level 2.", + "Also serves finished imports to peers on request, as bundles. The article IDs of your releases leave this node, and peers see which releases you have, tied to your node ID and IP address. Never shared as a bundle: rclone-encrypted files, files repaired by local PAR2 patches, and password-protected releases unless the password came inside the NZB itself on an import made after this feature was enabled.", }, ]; export const LEVEL_PRIVACY_NOTICE = "At this level other nodes can see which releases this instance has, tied to its node ID and IP address."; -// Level 3 adds nothing today (no bundles are served), so it carries the same -// notice as level 2 and claims nothing more. -export function levelPrivacyNotice(_level: number): string { - return LEVEL_PRIVACY_NOTICE; +export const LEVEL3_PRIVACY_NOTICE = + "At level 3 this node serves finished imports to peers on request, so the article IDs of your releases leave this node (at level 2 they do not). Peers see which releases you have, tied to your node ID and IP address. Never shared as a bundle: rclone-encrypted files, files repaired by local PAR2 patches, and password-protected releases unless the password came inside the NZB itself on an import made after this feature was enabled."; + +export function levelPrivacyNotice(level: number): string { + return level >= 3 ? LEVEL3_PRIVACY_NOTICE : LEVEL_PRIVACY_NOTICE; } // Levels 2 and 3 publish information about this instance's own releases. @@ -385,6 +386,7 @@ export function normalizePeernetConfig(raw: Partial | undefined): open_network: { level: level !== undefined && isPeernetLevel(level) ? level : 0, peers: [...(raw?.open_network?.peers ?? [])], + fast_path: raw?.open_network?.fast_path === true, }, sharing: { excluded_categories: [...(raw?.sharing?.excluded_categories ?? [])], diff --git a/frontend/src/components/queue/QueueItemCard.tsx b/frontend/src/components/queue/QueueItemCard.tsx index 373e13e3a..7de203a22 100644 --- a/frontend/src/components/queue/QueueItemCard.tsx +++ b/frontend/src/components/queue/QueueItemCard.tsx @@ -16,6 +16,7 @@ import { memo, useState } from "react"; import { formatBytes, formatRelativeTime, truncateText } from "../../lib/utils"; import { type QueueItem, QueueStatus } from "../../types/api"; import { PathDisplay } from "../ui/PathDisplay"; +import { PeerBadge } from "../ui/PeerBadge"; import { StatusBadge } from "../ui/StatusBadge"; interface QueueItemCardProps { @@ -71,6 +72,15 @@ export const QueueItemCard = memo(function QueueItemCard({ + {item.peer_origin && ( +
+ +
+ )} + {item.indexer && (
diff --git a/frontend/src/components/ui/PeerBadge.tsx b/frontend/src/components/ui/PeerBadge.tsx new file mode 100644 index 000000000..738b90359 --- /dev/null +++ b/frontend/src/components/ui/PeerBadge.tsx @@ -0,0 +1,23 @@ +import { peerBadge } from "../config/peernet/peernetStats"; + +interface PeerBadgeProps { + origin?: boolean; + fullCheckPending?: boolean; +} + +// Marks an item imported from a peer bundle. Renders nothing for other items, +// so rows without the flag keep their layout. +export function PeerBadge({ origin, fullCheckPending }: PeerBadgeProps) { + const info = peerBadge(origin, fullCheckPending); + if (!info) return null; + return ( + + {info.label} + + ); +} diff --git a/frontend/src/pages/HealthPage/components/HealthTable/HealthItemCard.tsx b/frontend/src/pages/HealthPage/components/HealthTable/HealthItemCard.tsx index 78adf0b84..56738e9a3 100644 --- a/frontend/src/pages/HealthPage/components/HealthTable/HealthItemCard.tsx +++ b/frontend/src/pages/HealthPage/components/HealthTable/HealthItemCard.tsx @@ -9,6 +9,7 @@ import { Wrench, } from "lucide-react"; import { memo, useMemo, useState } from "react"; +import { PeerBadge } from "../../../../components/ui/PeerBadge"; import { HealthBadge } from "../../../../components/ui/StatusBadge"; import { formatFutureTime, @@ -146,6 +147,10 @@ export const HealthItemCard = memo(function HealthItemCard({ {/* Quick Info Pills */}
+ {playbackImpact && } {errorDetails && } diff --git a/frontend/src/pages/HealthPage/components/HealthTable/HealthTableRow.tsx b/frontend/src/pages/HealthPage/components/HealthTable/HealthTableRow.tsx index 7afe84f15..daf1b2029 100644 --- a/frontend/src/pages/HealthPage/components/HealthTable/HealthTableRow.tsx +++ b/frontend/src/pages/HealthPage/components/HealthTable/HealthTableRow.tsx @@ -1,5 +1,6 @@ import { Clock, Heart, HeartCrack, Loader, Wrench } from "lucide-react"; import { memo, useMemo } from "react"; +import { PeerBadge } from "../../../../components/ui/PeerBadge"; import { HealthBadge } from "../../../../components/ui/StatusBadge"; import { formatFutureTime, @@ -135,6 +136,10 @@ export const HealthTableRow = memo(function HealthTableRow({
{item.file_path.split("/").pop() || ""}
+ {item.indexer && ( {item.indexer} diff --git a/frontend/src/pages/QueuePage.tsx b/frontend/src/pages/QueuePage.tsx index 8a05c39df..84eace5b7 100644 --- a/frontend/src/pages/QueuePage.tsx +++ b/frontend/src/pages/QueuePage.tsx @@ -35,6 +35,7 @@ import { ErrorAlert } from "../components/ui/ErrorAlert"; import { LoadingSpinner, LoadingTable } from "../components/ui/LoadingSpinner"; import { Pagination } from "../components/ui/Pagination"; import { PathDisplay } from "../components/ui/PathDisplay"; +import { PeerBadge } from "../components/ui/PeerBadge"; import { StatusBadge } from "../components/ui/StatusBadge"; import { useConfirm } from "../contexts/ModalContext"; import { useToast } from "../contexts/ToastContext"; @@ -885,6 +886,14 @@ export function QueuePage() { />
+ {item.peer_origin && ( +
+ +
+ )} {item.indexer && (
diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index 61033a036..824cd8e4c 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -29,6 +29,10 @@ export const QueueStatus = { export type QueueStatus = (typeof QueueStatus)[keyof typeof QueueStatus]; export interface QueueItem { + /** Imported from a peer bundle. */ + peer_origin?: boolean; + /** The full check of a peer-bundle import has not passed yet. */ + peer_full_check_pending?: boolean; id: number; nzb_path: string; nzb_display_name: string; @@ -197,6 +201,10 @@ export interface HealthCheckProgress { } export interface FileHealth { + /** Imported from a peer bundle. */ + peer_origin?: boolean; + /** The full check of a peer-bundle import has not passed yet. */ + peer_full_check_pending?: boolean; check_progress?: HealthCheckProgress; id: number; file_path: string; diff --git a/frontend/src/types/peernet.ts b/frontend/src/types/peernet.ts index faa89f477..43130bb88 100644 --- a/frontend/src/types/peernet.ts +++ b/frontend/src/types/peernet.ts @@ -8,6 +8,8 @@ export type PeernetLevel = 0 | 1 | 2 | 3; export interface PeernetOpenNetworkConfig { level: PeernetLevel; peers: string[]; + /** Use open-network layouts to skip archive analysis on NZB imports (default false). */ + fast_path: boolean; } export interface PeernetSharingConfig { @@ -101,12 +103,57 @@ export interface PeernetBackfillStatus { skipped: number; } -/** How many library files can be shared (those with a Release ID). */ +/** + * How many library files can be shared (those with a Release ID): either one + * from the release's NZB, or one derived from the files of an older import. + * The rest (file_id_only) could not be grouped: no_source_nzb, group_too_large + * or pending; not_indexable files could not be identified at all. + */ export interface PeernetLibraryStatus { files: number; with_release_id: number; + with_nzb_release_id: number; + with_derived_release_id: number; file_id_only: number; not_indexable: number; + no_source_nzb: number; + group_too_large: number; + pending: number; +} + +/** Bundle building, serving, fetching and importing since start. Every field is optional for an older backend; map keys are machine names. */ +export interface PeernetBundleImportStatus { + accepted?: number; + already_imported?: number; + rejected?: Record; + full_check_passed?: number; + full_check_failed?: number; + announced?: number; + bad_bundle_reports?: number; + /** Files that still owe their full check. */ + pending?: number; + /** Pending past the maximum age: failed, never announced. */ + stale?: number; +} + +export interface PeernetBundlesStatus { + built?: number; + cache_hits?: number; + build_failures?: number; + cached?: number; + cached_bytes?: number; + announced?: number; + not_shareable?: Record; + served?: number; + served_bytes?: number; + serve_refused?: Record; + fetched?: number; + fetch_rate_limited?: number; + fetch_failed?: Record; + fetch_backed_off?: number; + full_check_reports?: number; + full_check_backed_off?: number; + import?: PeernetBundleImportStatus; } export interface PeernetStatus { @@ -124,6 +171,7 @@ export interface PeernetStatus { library?: PeernetLibraryStatus; publish?: PeernetPublishStatus; advice?: PeernetAdviceStatus; + bundles?: PeernetBundlesStatus; } export type PeernetPeerSource = "user" | "exchange" | "inbound"; diff --git a/frontend/tests/peernetStats.test.ts b/frontend/tests/peernetStats.test.ts index d19d87f9e..a9a7e9763 100644 --- a/frontend/tests/peernetStats.test.ts +++ b/frontend/tests/peernetStats.test.ts @@ -1,14 +1,22 @@ import { describe, expect, test } from "bun:test"; import { backboneHint, + bundleHasActivity, describeBreakdown, + fetchFailLabel, formatRelativeTime, hasAnyPublishingLevel, - isLevelAvailable, + importRejectLabel, + levelNeedsConfirmation, libraryShareLine, + notShareableLabel, + peerBadge, publishedTotal, rejectionLabel, + serveRefusedLabel, showAdviceBlock, + showBundleServing, + showBundlesBlock, showPublishingDetails, showSharedData, skipLabel, @@ -17,6 +25,7 @@ import { EXPOSURE_LEVELS, levelPrivacyNotice, } from "../src/components/config/peernet/peernetValidation"; +import type { PeernetLibraryStatus } from "../src/types/peernet"; // Every key the backend can emit (internal/peernet/recsync/reasons.go, // internal/peernet/publish and internal/peernet/service/publishing.go). @@ -57,8 +66,140 @@ const BACKEND_SKIP_KEYS = [ "not_healthy", "reduced_by_peer_advice", "patch_status_unknown", + "peer_status_unknown", + "bundle_unavailable", ]; +// internal/peernet/bundle/build.go (ShareReason) and bundlex/builder.go. +const BACKEND_NOT_SHAREABLE_KEYS = [ + "rclone_encrypted", + "unsupported_encryption", + "password_not_from_nzb", + "healed_by_local_patch", + "known_holes", + "patch_status_unknown", + "no_nzb_segment_list", + "release_id_mismatch", + "invalid_metadata", + "too_large", + "no_post_date", + "no_files", + "self_check_failed", + "other", +]; +// internal/peernet/bundlex/serve.go +const BACKEND_SERVE_REFUSED_KEYS = [ + "unauthenticated", + "level", + "denied", + "backed_off", + "rate_limited", + "busy", + "error", + "unknown_bundle", + "excluded", + "unavailable", + "byte_budget", + "aborted", +]; +// internal/peernet/bundlex/fetch.go +const BACKEND_FETCH_FAILED_KEYS = [ + "refused", + "bad_status", + "transport", + "truncated", + "too_large", + "verification_failed", + "scheme_mismatch", + "membership", +]; +// internal/peernet/bundleimport (Reason) +const BACKEND_IMPORT_REJECT_KEYS = [ + "invalid", + "bad_name", + "too_large", + "too_many_files", + "health_disabled", + "bad_target", + "nzb_mismatch", + "collision", + "probe_missing", + "probe_inconclusive", + "no_media_file", + "media_invalid", + "media_missing", + "media_inconclusive", + "staging_failed", + "write_failed", + "unavailable", + "no_network", + "forbidden_file", + "sidecar_too_large", + "size_implausible", + "yenc_mismatch", + "structure_invalid", +]; + +describe("bundle reason coverage", () => { + test("every backend key has a label of its own", () => { + const groups: [readonly string[], (k: string) => string][] = [ + [BACKEND_NOT_SHAREABLE_KEYS, notShareableLabel], + [BACKEND_SERVE_REFUSED_KEYS, serveRefusedLabel], + [BACKEND_FETCH_FAILED_KEYS, fetchFailLabel], + [BACKEND_IMPORT_REJECT_KEYS, importRejectLabel], + ]; + for (const [keys, label] of groups) { + for (const key of keys) expect(label(key)).not.toBe(key); + } + }); + test("unknown and hostile keys are shown as text without touching prototypes", () => { + expect(notShareableLabel("brand_new")).toBe("brand_new"); + for (const k of ["__proto__", "constructor", "toString"]) { + expect(serveRefusedLabel(k)).toBe(k); + expect(importRejectLabel(k)).toBe(k); + } + }); +}); + +describe("bundle block visibility", () => { + test("block needs a receiving network; serving needs level 3", () => { + expect(showBundlesBlock([{ level: 0 }])).toBe(false); + expect(showBundlesBlock([{ level: 1 }])).toBe(true); + expect(showBundleServing([{ level: 2 }])).toBe(false); + expect(showBundleServing([{ level: 1 }, { level: 3 }])).toBe(true); + }); + test("activity detection ignores junk", () => { + expect(bundleHasActivity(undefined)).toBe(false); + expect(bundleHasActivity({ fetched: Number.NaN })).toBe(false); + expect(bundleHasActivity({ served: 1 })).toBe(true); + expect(bundleHasActivity({ import: { rejected: { invalid: 2 } } })).toBe(true); + }); +}); + +describe("level confirmation", () => { + test("only raising to level 3 needs confirmation", () => { + expect(levelNeedsConfirmation(2, 3)).toBe(true); + expect(levelNeedsConfirmation(0, 3)).toBe(true); + expect(levelNeedsConfirmation(3, 3)).toBe(false); + expect(levelNeedsConfirmation(3, 2)).toBe(false); + expect(levelNeedsConfirmation(0, 2)).toBe(false); + }); +}); + +describe("peer badge", () => { + test("no badge without the flag", () => { + expect(peerBadge(false, false)).toBeNull(); + expect(peerBadge(undefined, true)).toBeNull(); + expect(peerBadge(undefined, undefined)).toBeNull(); + }); + test("plain and verifying badges", () => { + expect(peerBadge(true, false)?.label).toBe("Peer network"); + const v = peerBadge(true, true); + expect(v?.label).toBe("Peer network \u00b7 verifying"); + expect(v?.title).toContain("announced to your ARRs and shared only after that check passes"); + }); +}); + describe("backend reason coverage", () => { test("every rejection reason has a label of its own", () => { for (const key of BACKEND_REJECTION_KEYS) { @@ -160,41 +301,110 @@ describe("publishedTotal", () => { }); describe("level honesty", () => { - test("level 3 is not available, 0-2 are", () => { - expect(EXPOSURE_LEVELS.map((l) => isLevelAvailable(l.level))).toEqual([ - true, - true, - true, - false, - ]); - expect(EXPOSURE_LEVELS[3]?.description).toContain("Not available yet"); - }); - test("level 2 text states the privacy consequence; no bundle claim", () => { + test("level 3 is described plainly with the privacy consequence", () => { + const d = EXPOSURE_LEVELS[3]?.description ?? ""; + expect(d).not.toContain("Not available"); + expect(d).not.toContain("behaves exactly like level 2"); + expect(d).toContain("article IDs"); + expect(d).toContain("node ID and IP address"); + }); + test("level 3 notice states that article IDs leave the node", () => { expect(levelPrivacyNotice(2)).toContain("node ID and IP address"); - expect(levelPrivacyNotice(3)).not.toContain("served"); + expect(levelPrivacyNotice(2)).not.toContain("article IDs leave"); + expect(levelPrivacyNotice(3)).toContain("article IDs"); + expect(levelPrivacyNotice(3)).toContain("rclone"); }); - test("level 1 states what it reveals", () => { + test("level 1 states what it reveals and that bundles can be requested", () => { expect(EXPOSURE_LEVELS[1]?.description).toContain("node ID and IP address"); + expect(EXPOSURE_LEVELS[1]?.description).toContain("bundle"); }); }); describe("library sharing line", () => { - test("says how many files can be shared when some cannot", () => { + const lib = (over: Partial): PeernetLibraryStatus => ({ + files: 0, + with_release_id: 0, + with_nzb_release_id: 0, + with_derived_release_id: 0, + file_id_only: 0, + not_indexable: 0, + no_source_nzb: 0, + group_too_large: 0, + pending: 0, + ...over, + }); + + test("says how many files can be shared and why the rest cannot", () => { expect( - libraryShareLine({ files: 2358, with_release_id: 41, file_id_only: 2317, not_indexable: 0 }), + libraryShareLine( + lib({ + files: 2358, + with_release_id: 2000, + with_nzb_release_id: 41, + with_derived_release_id: 1959, + file_id_only: 358, + no_source_nzb: 300, + group_too_large: 50, + pending: 8, + }), + ), ).toBe( - "41 of 2358 library files can be shared; the rest were imported before sharing data was kept.", + "2000 of 2358 library files can be shared (1959 of them are older imports, identified by the files they contain). " + + "The other 358 cannot be: 300 have no record of the import they came from, 50 are in imports with too many files, 8 are still being worked out.", ); }); - test("is silent when everything is shareable or nothing is known", () => { + test("explains older imports without listing reasons when only those are shared", () => { expect( - libraryShareLine({ files: 5, with_release_id: 5, file_id_only: 0, not_indexable: 0 }), - ).toBeNull(); + libraryShareLine( + lib({ + files: 12, + with_release_id: 10, + with_nzb_release_id: 4, + with_derived_release_id: 6, + file_id_only: 1, + no_source_nzb: 1, + not_indexable: 1, + }), + ), + ).toBe( + "10 of 12 library files can be shared (6 of them are older imports, identified by the files they contain). " + + "The other 2 cannot be: 1 has no record of the import it came from, 1 could not be read.", + ); + }); + test("keeps singular wording", () => { + expect( + libraryShareLine( + lib({ + files: 2, + with_release_id: 1, + with_nzb_release_id: 1, + file_id_only: 1, + group_too_large: 1, + }), + ), + ).toBe( + "1 of 2 library files can be shared. The other 1 cannot be: 1 is in an import with too many files.", + ); + }); + test("still reports a shortfall whose reason the server did not give", () => { expect( - libraryShareLine({ files: 0, with_release_id: 0, file_id_only: 0, not_indexable: 0 }), + libraryShareLine( + lib({ files: 5, with_release_id: 4, with_nzb_release_id: 4, file_id_only: 1 }), + ), + ).toBe("4 of 5 library files can be shared. The other 1 cannot be."); + }); + test("is silent when everything is shareable or nothing is known", () => { + expect( + libraryShareLine(lib({ files: 5, with_release_id: 5, with_nzb_release_id: 5 })), ).toBeNull(); + expect(libraryShareLine(lib({}))).toBeNull(); expect(libraryShareLine(undefined)).toBeNull(); }); + test("ignores nonsense numbers", () => { + expect(libraryShareLine(lib({ files: Number.NaN, with_release_id: 1 }))).toBeNull(); + expect(libraryShareLine(lib({ files: 3, with_release_id: -1 }))).toBeNull(); + expect(libraryShareLine(lib({ files: 3, with_release_id: 9 }))).toBeNull(); + }); }); describe("backbone hint", () => { diff --git a/frontend/tests/peernetValidation.test.ts b/frontend/tests/peernetValidation.test.ts index a22be94ea..c6152813a 100644 --- a/frontend/tests/peernetValidation.test.ts +++ b/frontend/tests/peernetValidation.test.ts @@ -151,6 +151,18 @@ describe("addTag", () => { }); }); +describe("open network fast path switch", () => { + test("defaults to off and only a literal true turns it on", () => { + expect(normalizePeernetConfig(undefined).open_network.fast_path).toBe(false); + const on = normalizePeernetConfig({ open_network: { level: 1, peers: [], fast_path: true } }); + expect(on.open_network.fast_path).toBe(true); + const odd = normalizePeernetConfig({ + open_network: { level: 1, peers: [], fast_path: "yes" as unknown as boolean }, + }); + expect(odd.open_network.fast_path).toBe(false); + }); +}); + describe("normalizePeernetConfig / validatePeernetConfig", () => { test("fills defaults and is valid", () => { const cfg = normalizePeernetConfig(undefined); diff --git a/internal/api/health_handlers.go b/internal/api/health_handlers.go index cac8db7e4..f3ae6bb23 100644 --- a/internal/api/health_handlers.go +++ b/internal/api/health_handlers.go @@ -106,6 +106,7 @@ func (s *Server) handleListHealth(c *fiber.Ctx) error { response[i].CheckProgress = s.healthWorker.GetCheckProgress(item.FilePath) } } + s.annotateHealthPeerOrigin(c.Context(), items, response) // Create metadata meta := &APIMeta{ @@ -167,6 +168,7 @@ func (s *Server) handleGetHealth(c *fiber.Ctx) error { if s.healthWorker != nil && item.Status == database.HealthStatusChecking { response.CheckProgress = s.healthWorker.GetCheckProgress(item.FilePath) } + s.annotateHealthPeerOrigin(c.Context(), []*database.FileHealth{item}, []*HealthItemResponse{response}) return RespondSuccess(c, response) } @@ -631,6 +633,7 @@ func (s *Server) handleListCorrupted(c *fiber.Ctx) error { response[i].CheckProgress = s.healthWorker.GetCheckProgress(item.FilePath) } } + s.annotateHealthPeerOrigin(c.Context(), corruptedItems, response) // Create metadata meta := &APIMeta{ diff --git a/internal/api/peer_bundle_origin.go b/internal/api/peer_bundle_origin.go new file mode 100644 index 000000000..284420864 --- /dev/null +++ b/internal/api/peer_bundle_origin.go @@ -0,0 +1,71 @@ +package api + +import ( + "context" + "log/slog" + "strings" + + "github.com/kipsilabs/altmount/internal/database" +) + +// PeerBundleLookup tells which queue items and files came from a peer bundle +// and whether their full check is still pending (the value is "pending"). +// *database.PeernetBundleImportRepository satisfies it. +type PeerBundleLookup interface { + StatesForQueues(ctx context.Context, ids []int64) (map[int64]bool, error) + StatesForPaths(ctx context.Context, paths []string) (map[string]bool, error) +} + +type peerBundleLookupHolder struct{ l PeerBundleLookup } + +// SetPeerBundleLookup installs (or, with nil, removes) the lookup. With none +// installed no query runs and every item reports false for both flags. +func (s *Server) SetPeerBundleLookup(l PeerBundleLookup) { + if l == nil { + s.peerBundles.Store(nil) + return + } + s.peerBundles.Store(&peerBundleLookupHolder{l: l}) +} + +func (s *Server) annotateQueuePeerOrigin(ctx context.Context, items []*database.ImportQueueItem, resp []*QueueItemResponse) { + h := s.peerBundles.Load() + if h == nil || len(items) == 0 { + return + } + ids := make([]int64, len(items)) + for i, it := range items { + ids[i] = it.ID + } + states, err := h.l.StatesForQueues(ctx, ids) + if err != nil { + slog.WarnContext(ctx, "Could not look up peer bundle imports", "error", err) + return + } + for i, it := range items { + if pending, ok := states[it.ID]; ok && i < len(resp) { + resp[i].PeerOrigin, resp[i].PeerFullCheckPending = true, pending + } + } +} + +func (s *Server) annotateHealthPeerOrigin(ctx context.Context, items []*database.FileHealth, resp []*HealthItemResponse) { + h := s.peerBundles.Load() + if h == nil || len(items) == 0 { + return + } + paths := make([]string, len(items)) + for i, it := range items { + paths[i] = it.FilePath + } + states, err := h.l.StatesForPaths(ctx, paths) + if err != nil { + slog.WarnContext(ctx, "Could not look up peer bundle imports", "error", err) + return + } + for i, it := range items { + if pending, ok := states[strings.Trim(strings.ReplaceAll(it.FilePath, `\`, "/"), "/")]; ok && i < len(resp) { + resp[i].PeerOrigin, resp[i].PeerFullCheckPending = true, pending + } + } +} diff --git a/internal/api/peer_bundle_origin_test.go b/internal/api/peer_bundle_origin_test.go new file mode 100644 index 000000000..dca4e59fa --- /dev/null +++ b/internal/api/peer_bundle_origin_test.go @@ -0,0 +1,75 @@ +package api + +import ( + "context" + "encoding/json" + "testing" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeLookup struct { + queues map[int64]bool + paths map[string]bool + calls int +} + +func (f *fakeLookup) StatesForQueues(_ context.Context, _ []int64) (map[int64]bool, error) { + f.calls++ + return f.queues, nil +} +func (f *fakeLookup) StatesForPaths(_ context.Context, _ []string) (map[string]bool, error) { + f.calls++ + return f.paths, nil +} + +func TestPeerOrigin_QueueItems(t *testing.T) { + s := &Server{} + items := []*database.ImportQueueItem{{ID: 1}, {ID: 2}, {ID: 3}} + resp := []*QueueItemResponse{ToQueueItemResponse(items[0]), ToQueueItemResponse(items[1]), ToQueueItemResponse(items[2])} + + s.annotateQueuePeerOrigin(context.Background(), items, resp) + for _, r := range resp { + assert.False(t, r.PeerOrigin || r.PeerFullCheckPending, "no lookup installed: nothing flagged") + } + + l := &fakeLookup{queues: map[int64]bool{1: true, 2: false}} + s.SetPeerBundleLookup(l) + s.annotateQueuePeerOrigin(context.Background(), items, resp) + assert.True(t, resp[0].PeerOrigin) + assert.True(t, resp[0].PeerFullCheckPending) + assert.True(t, resp[1].PeerOrigin) + assert.False(t, resp[1].PeerFullCheckPending, "verified") + assert.False(t, resp[2].PeerOrigin) + assert.Equal(t, 1, l.calls, "one lookup for the page") + + s.SetPeerBundleLookup(nil) + l.calls = 0 + s.annotateQueuePeerOrigin(context.Background(), items, resp) + assert.Zero(t, l.calls, "removed lookup is never asked") +} + +func TestPeerOrigin_HealthItemsMatchByNormalisedPath(t *testing.T) { + s := &Server{} + l := &fakeLookup{paths: map[string]bool{"movies/a.mkv": true}} + s.SetPeerBundleLookup(l) + items := []*database.FileHealth{{FilePath: "movies/a.mkv"}, {FilePath: "movies/b.mkv"}} + resp := []*HealthItemResponse{ToHealthItemResponse(items[0]), ToHealthItemResponse(items[1])} + s.annotateHealthPeerOrigin(context.Background(), items, resp) + assert.True(t, resp[0].PeerOrigin) + assert.True(t, resp[0].PeerFullCheckPending) + assert.False(t, resp[1].PeerOrigin) +} + +func TestPeerOrigin_JSONNames(t *testing.T) { + b, err := json.Marshal(ToQueueItemResponse(&database.ImportQueueItem{})) + require.NoError(t, err) + assert.Contains(t, string(b), `"peer_origin":false`) + assert.Contains(t, string(b), `"peer_full_check_pending":false`) + b, err = json.Marshal(ToHealthItemResponse(&database.FileHealth{})) + require.NoError(t, err) + assert.Contains(t, string(b), `"peer_origin":false`) + assert.Contains(t, string(b), `"peer_full_check_pending":false`) +} diff --git a/internal/api/peernet_handlers.go b/internal/api/peernet_handlers.go index fd5a1ab71..299e2a562 100644 --- a/internal/api/peernet_handlers.go +++ b/internal/api/peernet_handlers.go @@ -133,13 +133,21 @@ type peernetBackfillDTO struct { } // peernetLibraryDTO says how many library files can be shared: those with a -// Release ID. The others were imported before the release's NZB was kept -// (file_id_only) or could not be identified (not_indexable). +// Release ID, either one from the release's NZB (with_nzb_release_id) or one +// derived from the files of the import (with_derived_release_id, for imports +// that predate the NZB store). The others (file_id_only) could not be grouped: +// no_source_nzb, group_too_large or pending (the background pass has not +// reached them yet), or could not be identified at all (not_indexable). type peernetLibraryDTO struct { - Files int `json:"files"` - WithReleaseID int `json:"with_release_id"` - FileIDOnly int `json:"file_id_only"` - NotIndexable int `json:"not_indexable"` + Files int `json:"files"` + WithReleaseID int `json:"with_release_id"` + WithNZBReleaseID int `json:"with_nzb_release_id"` + WithDerivedReleaseID int `json:"with_derived_release_id"` + FileIDOnly int `json:"file_id_only"` + NotIndexable int `json:"not_indexable"` + NoSourceNZB int `json:"no_source_nzb"` + GroupTooLarge int `json:"group_too_large"` + Pending int `json:"pending"` } type peernetStatusDTO struct { @@ -158,6 +166,63 @@ type peernetStatusDTO struct { Library peernetLibraryDTO `json:"library"` Publish peernetPublishDTO `json:"publish"` Advice peernetAdviceDTO `json:"advice"` + Bundles peernetBundlesDTO `json:"bundles"` +} + +// peernetBundlesDTO counts bundle building, serving, fetching and importing +// since start. It holds no bundle content, path, name or address. +type peernetBundlesDTO struct { + Built int64 `json:"built"` + CacheHits int64 `json:"cache_hits"` + BuildFailures int64 `json:"build_failures"` + Cached int `json:"cached"` + CachedBytes int `json:"cached_bytes"` + Announced int `json:"announced"` + NotShareable map[string]int64 `json:"not_shareable"` + Served int64 `json:"served"` + ServedBytes int64 `json:"served_bytes"` + ServeRefused map[string]int64 `json:"serve_refused"` + Fetched int64 `json:"fetched"` + FetchRateLimited int64 `json:"fetch_rate_limited"` + FetchFailed map[string]int64 `json:"fetch_failed"` + FetchBackedOff int `json:"fetch_backed_off"` + FullCheckReports int64 `json:"full_check_reports"` + FullCheckBackedOff int `json:"full_check_backed_off"` + Import peernetImportDTO `json:"import"` +} + +type peernetImportDTO struct { + Accepted int64 `json:"accepted"` + AlreadyImported int64 `json:"already_imported"` + Rejected map[string]int64 `json:"rejected"` + FullCheckPassed int64 `json:"full_check_passed"` + FullCheckFailed int64 `json:"full_check_failed"` + Announced int64 `json:"announced"` + BadBundleReports int64 `json:"bad_bundle_reports"` + Pending int64 `json:"pending"` + Stale int64 `json:"stale"` +} + +func copyCounts(m map[string]int64) map[string]int64 { + out := make(map[string]int64, len(m)) + for k, v := range m { + out[k] = v + } + return out +} + +func toBundlesDTO(b service.BundleStatus) peernetBundlesDTO { + return peernetBundlesDTO{ + Built: b.Built, CacheHits: b.CacheHits, BuildFailures: b.BuildFailures, Cached: b.Cached, CachedBytes: b.CachedBytes, + Announced: b.Announced, NotShareable: copyCounts(b.NotShareable), Served: b.Served, ServedBytes: b.ServedBytes, + ServeRefused: copyCounts(b.ServeRefused), Fetched: b.Fetched, FetchRateLimited: b.FetchRateLimited, + FetchFailed: copyCounts(b.FetchFailed), FetchBackedOff: b.FetchBackedOff, + FullCheckReports: b.FullCheckReports, FullCheckBackedOff: b.FullCheckBackedOff, + Import: peernetImportDTO{Accepted: b.Import.Accepted, AlreadyImported: b.Import.AlreadyImported, + Rejected: copyCounts(b.Import.Rejected), FullCheckPassed: b.Import.FullCheckPassed, + FullCheckFailed: b.Import.FullCheckFailed, Announced: b.Import.Announced, BadBundleReports: b.Import.BadBundleReports, + Pending: b.Import.Pending, Stale: b.Import.Stale}, + } } // peernetPublishDTO counts what the publishing pipeline did since start. It @@ -383,9 +448,12 @@ func (s *Server) handlePeernetStatus(c *fiber.Ctx) error { Backfill: peernetBackfillDTO{Running: st.Backfill.Running, Scanned: st.Backfill.Scanned, Indexed: st.Backfill.Indexed, Skipped: st.Backfill.Skipped}, Library: peernetLibraryDTO{Files: st.Library.Files, WithReleaseID: st.Library.WithReleaseID, - FileIDOnly: st.Library.FileIDOnly, NotIndexable: st.Library.NotIndexable}, + WithNZBReleaseID: st.Library.WithNZBReleaseID, WithDerivedReleaseID: st.Library.WithDerivedReleaseID, + FileIDOnly: st.Library.FileIDOnly, NotIndexable: st.Library.NotIndexable, NoSourceNZB: st.Library.NoSourceNZB, + GroupTooLarge: st.Library.GroupTooLarge, Pending: st.Library.Pending}, Publish: toPublishDTO(st.Publish), Advice: toAdviceDTO(st.Advice), + Bundles: toBundlesDTO(st.Bundles), } for _, n := range st.Networks { out.Networks = append(out.Networks, toNetworkDTO(n)) diff --git a/internal/api/peernet_handlers_test.go b/internal/api/peernet_handlers_test.go index 9e5a03ba6..ee7a6ab2d 100644 --- a/internal/api/peernet_handlers_test.go +++ b/internal/api/peernet_handlers_test.go @@ -505,7 +505,8 @@ func TestPeernetStatusCarriesRecordsAndAdvice(t *testing.T) { func TestPeernetStatusReportsLibrarySharing(t *testing.T) { f := &fakePeernet{status: service.Status{Enabled: true, NodeID: hex64, - Library: service.LibraryStatus{Files: 10, WithReleaseID: 4, FileIDOnly: 5, NotIndexable: 1}}} + Library: service.LibraryStatus{Files: 10, WithReleaseID: 4, WithNZBReleaseID: 1, WithDerivedReleaseID: 3, FileIDOnly: 5, + NotIndexable: 1, NoSourceNZB: 2, GroupTooLarge: 1, Pending: 2}}} code, m, raw := doReq(t, peernetApp(f), "GET", "/api/peernet/status", "") require.Equal(t, 200, code, raw) lib, ok := m["data"].(map[string]any)["library"].(map[string]any) @@ -514,4 +515,42 @@ func TestPeernetStatusReportsLibrarySharing(t *testing.T) { assert.EqualValues(t, 4, lib["with_release_id"]) assert.EqualValues(t, 5, lib["file_id_only"]) assert.EqualValues(t, 1, lib["not_indexable"]) + assert.EqualValues(t, 1, lib["with_nzb_release_id"]) + assert.EqualValues(t, 3, lib["with_derived_release_id"]) + assert.EqualValues(t, 2, lib["no_source_nzb"]) + assert.EqualValues(t, 1, lib["group_too_large"]) + assert.EqualValues(t, 2, lib["pending"]) +} + +func TestPeernetStatusCarriesBundleCounters(t *testing.T) { + f := &fakePeernet{status: service.Status{Enabled: true, Bundles: service.BundleStatus{ + Built: 2, CacheHits: 1, BuildFailures: 3, Cached: 1, CachedBytes: 99, Announced: 4, + NotShareable: map[string]int64{"rclone_encrypted": 2}, Served: 5, ServedBytes: 600, + ServeRefused: map[string]int64{"rate_limited": 1}, Fetched: 7, FetchRateLimited: 2, + FetchFailed: map[string]int64{"verification_failed": 1}, FetchBackedOff: 1, FullCheckReports: 6, FullCheckBackedOff: 2, + Import: service.BundleImportStatus{Accepted: 8, AlreadyImported: 1, Rejected: map[string]int64{"probe_missing": 3}, + FullCheckPassed: 4, FullCheckFailed: 2, Announced: 3, BadBundleReports: 2}}}} + code, m, raw := doReq(t, peernetApp(f), "GET", "/api/peernet/status", "") + require.Equal(t, 200, code, raw) + b := m["data"].(map[string]any)["bundles"].(map[string]any) + for k, want := range map[string]float64{"built": 2, "cache_hits": 1, "build_failures": 3, "cached": 1, "cached_bytes": 99, + "announced": 4, "served": 5, "served_bytes": 600, "fetched": 7, "fetch_rate_limited": 2, "fetch_backed_off": 1, + "full_check_reports": 6, "full_check_backed_off": 2} { + assert.EqualValues(t, want, b[k], k) + } + assert.EqualValues(t, 2, b["not_shareable"].(map[string]any)["rclone_encrypted"]) + assert.EqualValues(t, 1, b["serve_refused"].(map[string]any)["rate_limited"]) + assert.EqualValues(t, 1, b["fetch_failed"].(map[string]any)["verification_failed"]) + im := b["import"].(map[string]any) + for k, want := range map[string]float64{"accepted": 8, "already_imported": 1, "full_check_passed": 4, + "full_check_failed": 2, "announced": 3, "bad_bundle_reports": 2} { + assert.EqualValues(t, want, im[k], k) + } + assert.EqualValues(t, 3, im["rejected"].(map[string]any)["probe_missing"]) + + // Nothing counted: stable shape, empty objects rather than null. + _, m, _ = doReq(t, peernetApp(&fakePeernet{status: service.Status{Enabled: true}}), "GET", "/api/peernet/status", "") + b = m["data"].(map[string]any)["bundles"].(map[string]any) + assert.NotNil(t, b["not_shareable"]) + assert.NotNil(t, b["import"].(map[string]any)["rejected"]) } diff --git a/internal/api/queue_handlers.go b/internal/api/queue_handlers.go index 977091e4a..953f325ca 100644 --- a/internal/api/queue_handlers.go +++ b/internal/api/queue_handlers.go @@ -19,6 +19,7 @@ import ( "github.com/kipsilabs/altmount/internal/database" internalerrors "github.com/kipsilabs/altmount/internal/errors" "github.com/kipsilabs/altmount/internal/httpclient" + "github.com/kipsilabs/altmount/internal/importer" "github.com/kipsilabs/altmount/internal/importer/utils/nzbtrim" "github.com/kipsilabs/altmount/internal/nzbfile" "github.com/kipsilabs/altmount/internal/nzblnk" @@ -170,6 +171,7 @@ func (s *Server) handleListQueue(c *fiber.Ctx) error { for i, item := range items { response[i] = ToQueueItemResponse(item) } + s.annotateQueuePeerOrigin(c.Context(), items, response) // Create metadata meta := &APIMeta{ @@ -218,6 +220,7 @@ func (s *Server) handleGetQueue(c *fiber.Ctx) error { // Convert to API response format response := ToQueueItemResponse(item) + s.annotateQueuePeerOrigin(c.Context(), []*database.ImportQueueItem{item}, []*QueueItemResponse{response}) return RespondSuccess(c, response) } @@ -311,6 +314,10 @@ func (s *Server) handleRetryQueue(c *fiber.Ctx) error { return RespondNotFound(c, "Queue item", "") } + if item.IsPeerImport() { + return RespondConflict(c, "Cannot retry a peer import", importer.ErrPeerImportNoNZB.Error()) + } + // Only allow retry of pending, failed or completed items if item.Status != database.QueueStatusPending && item.Status != database.QueueStatusFailed && item.Status != database.QueueStatusCompleted { return RespondConflict(c, "Can only retry pending, failed or completed items", "Current status: "+string(item.Status)) @@ -828,7 +835,7 @@ func (s *Server) handleUploadNZBLnk(c *fiber.Ctx) error { } } - item, err := s.importerService.AddToQueue(c.Context(), tempFile, basePath, categoryPtr, &priority, nil, nil, &resolved.Indexer) + item, err := s.importerService.AddToQueue(c.Context(), tempFile, basePath, categoryPtr, &priority, queueMetadataForPassword(resolved.Password), nil, &resolved.Indexer) if err != nil { os.Remove(tempFile) result.ErrorMessage = "Failed to add to queue: " + err.Error() @@ -870,6 +877,17 @@ func sanitizeFilename(name string) string { return result } +// queueMetadataForPassword is the queue metadata of an item whose NZB had a +// user-supplied password embedded by embedPasswordInNZB: it marks the password +// as injected (never copying it), so the importer does not take it for one the +// NZB came with. With no password there is no metadata. +func queueMetadataForPassword(password string) *string { + if password == "" { + return nil + } + return importer.MarkPasswordInjected(nil) +} + // embedPasswordInNZB injects a password into the NZB XML metadata func embedPasswordInNZB(nzbContent []byte, password string) []byte { if password == "" { @@ -981,7 +999,7 @@ func (s *Server) handleSearchNZBByName(c *fiber.Ctx) error { } } - item, err := s.importerService.AddToQueue(c.Context(), tempFile, basePath, categoryPtr, &priority, nil, nil, &resolved.Indexer) + item, err := s.importerService.AddToQueue(c.Context(), tempFile, basePath, categoryPtr, &priority, queueMetadataForPassword(resolved.Password), nil, &resolved.Indexer) if err != nil { os.Remove(tempFile) return RespondInternalError(c, "Failed to add to queue", err.Error()) @@ -1039,6 +1057,9 @@ func (s *Server) handleRestartQueueBulk(c *fiber.Ctx) error { if item.Status == database.QueueStatusProcessing { processedCount++ } + if item.IsPeerImport() { + return RespondConflict(c, "Cannot restart a peer import", importer.ErrPeerImportNoNZB.Error()) + } } if notFoundCount == len(request.IDs) { @@ -1380,6 +1401,10 @@ func (s *Server) handleDownloadNZB(c *fiber.Ctx) error { return RespondNotFound(c, "Queue item", "") } + if item.IsPeerImport() { + return RespondConflict(c, "No NZB for a peer import", importer.ErrPeerImportNoNZB.Error()) + } + resolved, resolveErr := nzbfile.ResolveOnDisk(item.NzbPath) if resolveErr != nil { // Raw .nzb is gone (deleted after successful import). diff --git a/internal/api/queue_password_origin_test.go b/internal/api/queue_password_origin_test.go new file mode 100644 index 000000000..a59e7ccbe --- /dev/null +++ b/internal/api/queue_password_origin_test.go @@ -0,0 +1,17 @@ +package api + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInjectedPasswordQueueMetadata(t *testing.T) { + assert.Nil(t, queueMetadataForPassword(""), "no password, no injection, no metadata (the queue row stays as it was)") + m := queueMetadataForPassword("typed-secret") + require.NotNil(t, m) + assert.Contains(t, *m, "_altmount_pw_injected") + assert.False(t, strings.Contains(*m, "typed-secret"), "the password is never copied into the metadata") +} diff --git a/internal/api/queue_peer_import_test.go b/internal/api/queue_peer_import_test.go new file mode 100644 index 000000000..7dc2f02f4 --- /dev/null +++ b/internal/api/queue_peer_import_test.go @@ -0,0 +1,61 @@ +package api + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gofiber/fiber/v2" + "github.com/kipsilabs/altmount/internal/database" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The queue row of a direct peer import has no NZB behind it: retry, restart +// and download must refuse it with a clear message and change nothing. +func TestQueuePeerImport_NzbActionsAreRefusedCleanly(t *testing.T) { + server, repo, _ := newSABnzbdHistoryTestServer(t) + ctx := context.Background() + item := &database.ImportQueueItem{NzbPath: database.PeerImportNzbPrefix + "abc/Release.nzb", Status: database.QueueStatusCompleted, + Priority: database.QueuePriorityNormal} + require.NoError(t, repo.AddToQueue(ctx, item)) + + app := fiber.New() + app.Post("/queue/:id/retry", server.handleRetryQueue) + app.Post("/queue/bulk/restart", server.handleRestartQueueBulk) + app.Get("/queue/:id/download", server.handleDownloadNZB) + + for name, req := range map[string]func() *fiberReq{ + "retry": func() *fiberReq { return newReq("POST", "/queue/1/retry", "") }, + "restart": func() *fiberReq { return newReq("POST", "/queue/bulk/restart", `{"ids":[1]}`) }, + "download": func() *fiberReq { return newReq("GET", "/queue/1/download", "") }, + } { + t.Run(name, func(t *testing.T) { + r := req() + resp, err := app.Test(r.build()) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, 409, resp.StatusCode) + body, _ := io.ReadAll(resp.Body) + assert.Contains(t, string(body), "peer bundle") + got, err := repo.GetQueueItem(ctx, item.ID) + require.NoError(t, err) + assert.Equal(t, database.QueueStatusCompleted, got.Status, "nothing changed") + }) + } +} + +type fiberReq struct{ method, url, body string } + +func newReq(method, url, body string) *fiberReq { return &fiberReq{method, url, body} } + +func (r *fiberReq) build() *http.Request { + req := httptest.NewRequest(r.method, r.url, strings.NewReader(r.body)) + if r.body != "" { + req.Header.Set("Content-Type", "application/json") + } + return req +} diff --git a/internal/api/sabnzbd_handlers.go b/internal/api/sabnzbd_handlers.go index 3b3bb8e07..4f343018d 100644 --- a/internal/api/sabnzbd_handlers.go +++ b/internal/api/sabnzbd_handlers.go @@ -634,7 +634,7 @@ func (s *Server) handleSABnzbdQueue(c *fiber.Ctx) error { } // Get pending and processing items - items, err := s.queueRepo.ListActiveQueueItems(c.Context(), "", categoryFilter, limit, start, "updated_at", "desc") + items, err := s.queueRepo.ListActiveQueueItemsWithHeld(c.Context(), "", categoryFilter, limit, start, "updated_at", "desc") if err != nil { return s.writeSABnzbdErrorFiber(c, "Failed to get queue") } @@ -642,7 +642,7 @@ func (s *Server) handleSABnzbdQueue(c *fiber.Ctx) error { // Get total count for pagination info // Let's just use a simple approach for now: - totalCount, err := s.queueRepo.CountActiveQueueItems(c.Context(), "", categoryFilter) + totalCount, err := s.queueRepo.CountActiveQueueItemsWithHeld(c.Context(), "", categoryFilter) if err != nil { totalCount = len(items) // Fallback } @@ -657,7 +657,12 @@ func (s *Server) handleSABnzbdQueue(c *fiber.Ctx) error { continue } - slot := ToSABnzbdQueueSlot(item, start+i, s.progressBroadcaster) + var slot SABnzbdQueueSlot + if item.PeerHold { + slot = toSABnzbdHeldPeerSlot(item, start+i, s.progressBroadcaster) + } else { + slot = ToSABnzbdQueueSlot(item, start+i, s.progressBroadcaster) + } slots = append(slots, slot) if mb, err := strconv.ParseFloat(slot.Mb, 64); err == nil { @@ -953,7 +958,7 @@ func (s *Server) respondSABnzbdHistoryByIDs( // 1. Numeric id → queue first, then history.nzb_id. if numericID, err := strconv.ParseInt(id, 10, 64); err == nil { if item, err := s.queueRepo.GetQueueItem(ctx, numericID); err == nil && item != nil { - if !item.SkipArrNotification && categoryMatches(item.Category) && !seen[item.ID] { + if !item.SkipArrNotification && !item.PeerHold && categoryMatches(item.Category) && !seen[item.ID] { finalItems = append(finalItems, item) if item.Status == database.QueueStatusCompleted { liveCompleted[item] = true @@ -973,7 +978,7 @@ func (s *Server) respondSABnzbdHistoryByIDs( // 2. String id → queue.download_id, then history.download_id. if item, err := s.queueRepo.GetQueueItemByDownloadID(ctx, id); err == nil && item != nil { - if !item.SkipArrNotification && categoryMatches(item.Category) && !seen[item.ID] { + if !item.SkipArrNotification && !item.PeerHold && categoryMatches(item.Category) && !seen[item.ID] { finalItems = append(finalItems, item) if item.Status == database.QueueStatusCompleted { liveCompleted[item] = true diff --git a/internal/api/sabnzbd_peer_hold_test.go b/internal/api/sabnzbd_peer_hold_test.go new file mode 100644 index 000000000..fedb3950d --- /dev/null +++ b/internal/api/sabnzbd_peer_hold_test.go @@ -0,0 +1,136 @@ +package api + +import ( + "context" + "encoding/json" + "io" + "net/http/httptest" + "testing" + "time" + + "github.com/gofiber/fiber/v2" + "github.com/kipsilabs/altmount/internal/database" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A release imported from a peer bundle is playable at once but owes a full +// health check. Until it passes, the SABnzbd-compatible API shows it in the +// queue as work in progress and not in the history (so the ARR does not import +// unverified content); a pass moves it to the history as completed, a failure +// (the full check failed, or the 7-day limit) as failed. + +type peerHoldEnv struct { + server *Server + repo *database.Repository + state *database.PeernetBundleImportRepository + id int64 + dlID string +} + +func newPeerHoldEnv(t *testing.T, skip bool) *peerHoldEnv { + t.Helper() + server, repo, conn := newSABnzbdHistoryTestServer(t) + id := addHistoryQueueRow(t, repo, conn, "peer", database.QueueStatusCompleted, "tv", "nzo_peer", time.Now().UTC().Add(-time.Minute)) + if skip { + _, err := conn.Exec(`UPDATE import_queue SET skip_arr_notification = 1 WHERE id = ?`, id) + require.NoError(t, err) + } + size := int64(1 << 30) + _, err := conn.Exec(`UPDATE import_queue SET file_size = ?, storage_path = '/complete/tv/peer' WHERE id = ?`, size, id) + require.NoError(t, err) + state := database.NewPeernetBundleImportRepository(conn, database.DialectSQLite) + require.NoError(t, state.SetQueueHold(context.Background(), id, true)) + return &peerHoldEnv{server: server, repo: repo, state: state, id: id, dlID: "nzo_peer"} +} + +func (e *peerHoldEnv) queue(t *testing.T) SABnzbdQueueObject { + t.Helper() + app := fiber.New() + app.Get("/sabnzbd", e.server.handleSABnzbdQueue) + resp, err := app.Test(httptest.NewRequest("GET", "/sabnzbd?mode=queue&category=tv", nil)) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + var out SABnzbdQueueResponse + require.NoError(t, json.Unmarshal(body, &out), "body: %s", body) + return out.Queue +} + +func queueIDs(q SABnzbdQueueObject) []string { + ids := make([]string, 0, len(q.Slots)) + for _, s := range q.Slots { + ids = append(ids, s.NzoID) + } + return ids +} + +func (e *peerHoldEnv) history(t *testing.T, query string) []SABnzbdHistorySlot { + t.Helper() + return getSABnzbdHistory(t, e.server, "category=tv"+query).Slots +} + +func TestSABnzbdPeerHold_PendingIsInTheQueueAsWorkInProgressNotInHistory(t *testing.T) { + e := newPeerHoldEnv(t, false) + + q := e.queue(t) + require.Equal(t, []string{e.dlID}, queueIDs(q)) + assert.Equal(t, "Downloading", q.Slots[0].Status, "an existing in-progress status, never a terminal one") + assert.Equal(t, "99", q.Slots[0].Percentage) + assert.Equal(t, 1, q.Noofslots) + + assert.Empty(t, e.history(t, ""), "unverified content is not in the history") + assert.Empty(t, e.history(t, "&nzo_ids="+e.dlID), "nor when asked for by id") + assert.Empty(t, e.history(t, "&nzo_ids=1"), "nor by numeric id") +} + +func TestSABnzbdPeerHold_PassedMovesToHistoryAsCompleted(t *testing.T) { + e := newPeerHoldEnv(t, false) + require.NoError(t, e.state.SetQueueHold(context.Background(), e.id, false)) + + assert.Empty(t, queueIDs(e.queue(t))) + h := e.history(t, "") + require.Len(t, h, 1) + assert.Equal(t, e.dlID, h[0].NzoID) + assert.Equal(t, "Completed", h[0].Status) + assert.Len(t, e.history(t, "&nzo_ids="+e.dlID), 1) +} + +func TestSABnzbdPeerHold_FailedAndStaleAreFailedInHistoryWithAMessage(t *testing.T) { + for name, msg := range map[string]string{ + "check failed": "Peer-supplied release failed its full health check and was removed", + "stale": "Peer-supplied release was not fully verified within 7 days and was removed", + } { + t.Run(name, func(t *testing.T) { + e := newPeerHoldEnv(t, false) + changed, err := e.state.FailHeldQueueItem(context.Background(), e.id, msg) + require.NoError(t, err) + require.True(t, changed) + + assert.Empty(t, queueIDs(e.queue(t))) + h := e.history(t, "") + require.Len(t, h, 1) + assert.Equal(t, "Failed", h[0].Status) + assert.Equal(t, msg, h[0].Fail_message) + byID := e.history(t, "&nzo_ids="+e.dlID) + require.Len(t, byID, 1) + assert.Equal(t, "Failed", byID[0].Status) + }) + } +} + +func TestSABnzbdPeerHold_UserSkipStaysHiddenInEveryState(t *testing.T) { + e := newPeerHoldEnv(t, true) + assert.Empty(t, queueIDs(e.queue(t)), "held and skipped by the user: not in the queue") + assert.Empty(t, e.history(t, "")) + + require.NoError(t, e.state.SetQueueHold(context.Background(), e.id, false)) + assert.Empty(t, e.history(t, ""), "passed: still hidden") + assert.Empty(t, e.history(t, "&nzo_ids="+e.dlID)) + + require.NoError(t, e.state.SetQueueHold(context.Background(), e.id, true)) + _, err := e.state.FailHeldQueueItem(context.Background(), e.id, "x") + require.NoError(t, err) + assert.Empty(t, e.history(t, ""), "failed: still hidden") +} diff --git a/internal/api/sabnzbd_types.go b/internal/api/sabnzbd_types.go index 4e8af1a8a..076156ecb 100644 --- a/internal/api/sabnzbd_types.go +++ b/internal/api/sabnzbd_types.go @@ -392,6 +392,30 @@ func ToSABnzbdQueueSlot(item *database.ImportQueueItem, index int, progressBroad } } +// toSABnzbdHeldPeerSlot is the queue slot of a completed import that came from +// a peer bundle and still owes its full health check. It is reported with the +// status this API already uses for work in progress ("Downloading", which every +// ARR client treats as active) at 99 percent with 1 percent left, never as +// complete: the ARR must not import unverified content. When the check passes +// the item leaves the queue and appears in the history as completed; when it +// fails it appears there as failed. +func toSABnzbdHeldPeerSlot(item *database.ImportQueueItem, index int, pb *progress.ProgressBroadcaster) SABnzbdQueueSlot { + working := *item + working.Status = database.QueueStatusProcessing + slot := ToSABnzbdQueueSlot(&working, index, pb) + var total int64 + if item.FileSize != nil { + total = *item.FileSize + } + left := total / 100 + slot.Percentage = "99" + slot.Sizeleft = formatHumanSize(left) + slot.Mbleft = formatSizeMB(left) + slot.Timeleft = "0:00:01" + slot.Eta = "unknown" + return slot +} + // ToSABnzbdHistorySlot converts an AltMount ImportQueueItem to SABnzbd history format func ToSABnzbdHistorySlot(item *database.ImportQueueItem, index int, finalPath string) SABnzbdHistorySlot { if item == nil { diff --git a/internal/api/server.go b/internal/api/server.go index b50eab872..989bd717b 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -52,7 +52,15 @@ type Server struct { peernet PeernetService // networkLabeler labels library streams healthy on the peer network; nil // (the default) leaves Stremio responses untouched. - networkLabeler atomic.Pointer[networkLabelerHolder] + networkLabeler atomic.Pointer[networkLabelerHolder] + // peerPlayback lists and plays Stremio candidates from the peer network; nil + // while the feature is off (the stream list is then exactly as before). + peerPlayback atomic.Pointer[peerPlaybackHolder] + peerPlayGroup singleflight.Group + peerPlayLimits atomic.Pointer[peerPlayLimiter] + // peerBundles marks queue and health items that came from a peer bundle; + // nil (the default) means no lookup and both flags false. + peerBundles atomic.Pointer[peerBundleLookupHolder] metadataReader *metadata.MetadataReader metadataService *metadata.MetadataService nzbFilesystem *nzbfilesystem.NzbFilesystem @@ -201,6 +209,7 @@ func (s *Server) SetupRoutes(app *fiber.App) { stremioGroup.Get("/:key/manifest.json", s.handleStremioManifest) stremioGroup.Get("/:key/stream/:type/:id.json", s.handleStremioAddonStream) stremioGroup.Get("/:key/play", s.handleStremioAddonPlay) + stremioGroup.Get("/:key/peer/:release", s.handleStremioPeerPlay) api := app.Group(s.config.Prefix) diff --git a/internal/api/stremio_addon_handlers.go b/internal/api/stremio_addon_handlers.go index fb3b27ab1..1f500644f 100644 --- a/internal/api/stremio_addon_handlers.go +++ b/internal/api/stremio_addon_handlers.go @@ -240,6 +240,10 @@ func (s *Server) handleStremioAddonStream(c *fiber.Ctx) error { } } libraryStreams = s.labelNetworkHealthy(ctx, libraryStreams) + // Releases other nodes announced for this content with a bundle this node + // can fetch (nil with the peer network off: nothing is added or asked). + // They are looked up while the indexer search runs, never before it. + waitPeerStreams := s.startPeerCandidates(ctx, cfg, streamType, imdbID, baseURL, key, season, episode) // Map Stremio type to Prowlarr search type prowlarrType := "search" @@ -254,7 +258,8 @@ func (s *Server) handleStremioAddonStream(c *fiber.Ctx) error { "type", streamType, "id", rawID, "imdb_id", imdbID, "library_matches", len(libraryStreams)) results, cachedItems, err := s.searchStremioReleases(ctx, cfg, streamType, prowlarrType, imdbID, season, episode) - if err != nil && len(libraryStreams) == 0 { + peerStreams := waitPeerStreams() + if err != nil && len(libraryStreams) == 0 && len(peerStreams) == 0 { return emptyStreamsResponse(c) } @@ -262,17 +267,7 @@ func (s *Server) handleStremioAddonStream(c *fiber.Ctx) error { baseURL, key, streamType, season, episode, imdbID, resolveStreamDisplay(cfg.Stremio), s.storeStremioReleaseRef) - streams := make([]fiber.Map, 0, len(libraryStreams)+len(entries)) - streams = append(streams, libraryStreams...) - - for _, e := range entries { - streams = append(streams, fiber.Map{ - "name": e.Name, - "title": e.Title, - "url": e.URL, - "behaviorHints": e.BehaviorHints, - }) - } + streams := assembleStremioStreams(libraryStreams, peerStreams, entries) if len(streams) == 0 { return emptyStreamsResponse(c) diff --git a/internal/api/stremio_peer.go b/internal/api/stremio_peer.go new file mode 100644 index 000000000..99ccc705f --- /dev/null +++ b/internal/api/stremio_peer.go @@ -0,0 +1,393 @@ +package api + +import ( + "context" + "encoding/json" + "log/slog" + "net/url" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "time" + "unicode" + + "github.com/gofiber/fiber/v2" + "github.com/google/uuid" + "github.com/kipsilabs/altmount/internal/config" + "github.com/kipsilabs/altmount/internal/peernet/service" + "github.com/kipsilabs/altmount/internal/prowlarr" +) + +// PeerPlayRequest asks for a release to be imported from the peer network. +type PeerPlayRequest struct { + ReleaseID string + Category string + Metadata string + DownloadID string +} + +// PeerPlayResult is an imported release's files (virtual paths and sizes). +type PeerPlayResult struct { + Paths []string + Sizes []int64 +} + +// PeerPlayback is the peer network's part of the Stremio addon. +type PeerPlayback interface { + Candidates(ctx context.Context, imdbID string, season, episode *int) []service.PeerCandidate + HasOffer(ctx context.Context, releaseID string) bool + Play(ctx context.Context, req PeerPlayRequest) (PeerPlayResult, error) +} + +type peerPlaybackHolder struct{ p PeerPlayback } + +// SetPeerPlayback installs (or, with nil, removes) the peer network's Stremio +// support. With none installed the stream list is exactly what it was without +// the peer network, the play route answers 404 and no lookup runs. +func (s *Server) SetPeerPlayback(p PeerPlayback) { + if p == nil { + s.peerPlayback.Store(nil) + return + } + s.peerPlayback.Store(&peerPlaybackHolder{p: p}) +} + +const ( + // maxPeerStreams bounds the peer candidates of one stream list. + maxPeerStreams = 10 + // peerStreamMarker is the fixed local marker of a peer candidate. Nothing a + // peer sent is ever part of it. + peerStreamMarker = "🌐 Peer network" + // peerPlayTimeout bounds one peer import (fetch, probes, write), which + // normally takes a few seconds. + peerPlayTimeout = 2 * time.Minute + // maxPeerTitleRunes caps a peer title before it is rendered (records cap it + // too; this keeps the listing safe whatever the store holds). + maxPeerTitleRunes = 200 +) + +var ( + peerReleaseIDRe = regexp.MustCompile(`^[0-9a-f]{64}$`) + peerIMDbRe = regexp.MustCompile(`^tt[0-9]{1,10}$`) +) + +// peerText makes a peer-supplied title safe to show as plain text: control +// characters and line breaks are dropped and the length is capped. It is only +// ever placed inside a JSON string value, never in a URL or a path. +func peerText(s string) string { + var b strings.Builder + n := 0 + for _, r := range s { + if unicode.IsControl(r) || r == '\u2028' || r == '\u2029' { + continue + } + if n++; n > maxPeerTitleRunes { + break + } + b.WriteRune(r) + } + return strings.TrimSpace(b.String()) +} + +// peerCandidateStreams lists the peer network's candidates for a media key as +// Stremio streams. It returns nothing with the feature off, with no season and +// episode for a series, or on any failure. +func (s *Server) peerCandidateStreams(ctx context.Context, cfg *config.Config, streamType, imdbID, baseURL, key string, season, episode int) []fiber.Map { + h := s.peerPlayback.Load() + if h == nil || !peerIMDbRe.MatchString(imdbID) { + return nil + } + var sp, ep *int + if streamType == "series" { + if season <= 0 || episode <= 0 { + return nil + } + sp, ep = &season, &episode + } + cands := h.p.Candidates(ctx, imdbID, sp, ep) + display := resolveStreamDisplay(cfg.Stremio) + var out []fiber.Map + for _, c := range cands { + if len(out) >= maxPeerStreams { + break + } + title := peerText(c.Title) + if !peerReleaseIDRe.MatchString(c.ReleaseID) || title == "" { + continue + } + name, description := formatStremioStream(display, prowlarr.NZBResult{Title: title, Size: c.TotalSize, Indexer: peerStreamMarker}, false) + description += "\n" + peerStreamMarker + " • verified on play" + if c.Healthy { + description += "\n" + stremioNetworkHealthyLabel + } + playURL := baseURL + "/stremio/" + key + "/peer/" + c.ReleaseID + "?type=" + url.QueryEscape(streamType) + "&id=" + url.QueryEscape(imdbID) + if sp != nil { + playURL += "&season=" + strconv.Itoa(season) + "&episode=" + strconv.Itoa(episode) + } + out = append(out, fiber.Map{ + "name": name, "title": description, "url": playURL, + "behaviorHints": stremioStreamBehaviorHints{Service: "altmount", Cached: false, Filename: title, Indexer: peerStreamMarker}, + }) + } + return out +} + +const ( + // peerCandidateBudget bounds the whole peer lookup of one listing. + peerCandidateBudget = 3 * time.Second + // peerCandidateGrace is how long a listing waits for the peer lookup after + // its indexer search finished: a slow or hung lookup never delays the + // response by more than this. + peerCandidateGrace = 250 * time.Millisecond +) + +// startPeerCandidates starts the peer lookup of a listing in the background and +// returns the function that collects it. The lookup runs next to the indexer +// search; collecting waits at most peerCandidateGrace for a lookup that is still +// running and then gives up (the lookup is cancelled). It fails open: a panic or +// a slow lookup yields no peer streams, never an error or a delay. With the +// feature off nothing is started. +func (s *Server) startPeerCandidates(ctx context.Context, cfg *config.Config, streamType, imdbID, baseURL, key string, season, episode int) func() []fiber.Map { + if s.peerPlayback.Load() == nil { + return func() []fiber.Map { return nil } + } + lookupCtx, cancel := context.WithTimeout(ctx, peerCandidateBudget) + done := make(chan []fiber.Map, 1) + go func() { + defer func() { + if r := recover(); r != nil { + slog.ErrorContext(ctx, "peer candidate lookup panicked", "panic", r) + done <- nil + } + }() + done <- s.peerCandidateStreams(lookupCtx, cfg, streamType, imdbID, baseURL, key, season, episode) + }() + return func() []fiber.Map { + defer cancel() + select { + case out := <-done: + return out + default: + } + timer := time.NewTimer(peerCandidateGrace) + defer timer.Stop() + select { + case out := <-done: + return out + case <-timer.C: + return nil + } + } +} + +// assembleStremioStreams merges the parts of a listing in their fixed order: +// library streams, peer streams, indexer entries. With no peer streams the +// result is exactly what a listing without the peer network contains. +func assembleStremioStreams(library, peer []fiber.Map, entries []stremioStreamEntry) []fiber.Map { + streams := make([]fiber.Map, 0, len(library)+len(peer)+len(entries)) + streams = append(streams, library...) + streams = append(streams, peer...) + for _, e := range entries { + streams = append(streams, fiber.Map{ + "name": e.Name, + "title": e.Title, + "url": e.URL, + "behaviorHints": e.BehaviorHints, + }) + } + return streams +} + +// pickPeerStreamPath chooses the file to play from an imported release: the +// largest media file that is not a sample, and, for an episode, one that +// matches it. +func pickPeerStreamPath(res PeerPlayResult, selector *stremioEpisodeSelector) (string, bool) { + best, bestSize := "", int64(-1) + for i, p := range res.Paths { + if !isMediaExtension(filepath.Ext(p)) || isSampleFile(p) { + continue + } + if selector != nil && !selector.matches(p) { + continue + } + var size int64 + if i < len(res.Sizes) { + size = res.Sizes[i] + } + if size > bestSize { + best, bestSize = p, size + } + } + return best, best != "" +} + +// handleStremioPeerPlay handles GET /stremio/:key/peer/:release: it imports a +// release from the peer network (fetch, verify, probes, write: a few seconds) +// and redirects to its stream like a freshly imported NZB. The route is key +// authenticated like the rest of the addon. The release must be 64 lower-case +// hex characters and one that a stored signed record offers a fetchable bundle +// for; the address dialled comes only from this node's verified peer table. +func (s *Server) handleStremioPeerPlay(c *fiber.Ctx) error { + ctx := c.Context() + if s.configManager == nil { + return RespondServiceUnavailable(c, "Configuration not available", "") + } + cfg := s.configManager.GetConfig() + h := s.peerPlayback.Load() + if !isStremioEnabled(cfg) || h == nil { + return RespondNotFound(c, "Peer network endpoint", "") + } + key := c.Params("key") + if !s.validateDownloadKey(ctx, key) { + return RespondUnauthorized(c, "Invalid key", "") + } + releaseID := c.Params("release") + if !peerReleaseIDRe.MatchString(releaseID) { + return RespondBadRequest(c, "Invalid release", "") + } + streamType := c.Query("type") + imdbID := c.Query("id") + if (streamType != "movie" && streamType != "series") || !peerIMDbRe.MatchString(imdbID) { + return RespondBadRequest(c, "Invalid content", "") + } + selector := stremioEpisodeSelectorFromRequest(c) + if streamType == "series" && selector == nil { + return RespondBadRequest(c, "A season and an episode are required", "") + } + if retry, ok := s.peerLimiter().allow(key, time.Now()); !ok { + c.Set("Retry-After", strconv.Itoa(retry)) + return RespondError(c, fiber.StatusTooManyRequests, "RATE_LIMITED", "Too many peer plays", "try again shortly") + } + if !h.p.HasOffer(ctx, releaseID) { + return RespondNotFound(c, "Peer release", "") + } + + category, season, episode := "Movies", 0, 0 + if streamType == "series" { + category = "TV" + } + if selector != nil { + season, episode = selector.Season, selector.Episode + } + // The media ids come from the request, not from anything verified: they + // are kept for this node's own listing under distinct "unverified_" keys, + // which the publisher's media-id reader does not parse, so a peer's claim + // is never republished in this node's signed records. + contentKey := stremioContentKey(streamType, imdbID, selector) + data, err := json.Marshal(peerQueueMetadata{ + ContentKey: contentKey, ReleaseKey: "peernet:" + releaseID, + UnverifiedIMDbID: imdbID, UnverifiedType: streamType, UnverifiedSeason: season, UnverifiedEpisode: episode, + }) + if err != nil { + return RespondInternalError(c, "Failed to prepare peer release", "") + } + meta := new(string) + *meta = string(data) + req := PeerPlayRequest{ReleaseID: releaseID, Category: category, Metadata: *meta, DownloadID: stremioDownloadIDPrefix + uuid.NewString()} + // Concurrent plays of one release share one import; it is detached from the + // caller so one client leaving does not abort the others' work. + v, err, _ := s.peerPlayGroup.Do(releaseID+"|"+contentKey, func() (any, error) { + workCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), peerPlayTimeout) + defer cancel() + return h.p.Play(workCtx, req) + }) + if err != nil { + slog.WarnContext(ctx, "Peer network play failed", "error", err) + return RespondServiceUnavailable(c, "Failed to prepare peer release", "the release could not be verified here") + } + res, _ := v.(PeerPlayResult) + path, ok := pickPeerStreamPath(res, selector) + if !ok { + return RespondServiceUnavailable(c, "Failed to prepare peer release", "no playable media file in the release") + } + baseURL := resolveBaseURL(c, cfg.Stremio.BaseURL) + return c.Redirect(baseURL+"/api/files/stream?path="+url.QueryEscape(path)+"&download_key="+url.QueryEscape(key), fiber.StatusFound) +} + +// peerPlayLimiter bounds peer plays, which trigger an outbound fetch and +// article probes: a token bucket per key and one for all keys. +type peerPlayLimiter struct { + mu sync.Mutex + keyRate, globalRate float64 // tokens per second + keyBurst, globalBurst float64 + keys map[string]*peerBucket + global peerBucket +} + +type peerBucket struct { + tokens float64 + at time.Time + init bool +} + +const maxPeerLimiterKeys = 1024 + +func newPeerPlayLimiter(perKeyPerMin, perKeyBurst, globalPerMin, globalBurst int) *peerPlayLimiter { + return &peerPlayLimiter{ + keyRate: float64(perKeyPerMin) / 60, keyBurst: float64(perKeyBurst), + globalRate: float64(globalPerMin) / 60, globalBurst: float64(globalBurst), + keys: map[string]*peerBucket{}, + } +} + +func (b *peerBucket) take(now time.Time, rate, burst float64) bool { + if !b.init { + b.tokens, b.at, b.init = burst, now, true + } + b.tokens = min(burst, b.tokens+now.Sub(b.at).Seconds()*rate) + b.at = now + if b.tokens < 1 { + return false + } + b.tokens-- + return true +} + +// allow spends one token of the key's bucket and one of the global bucket. On +// refusal it returns the seconds to wait. +func (l *peerPlayLimiter) allow(key string, now time.Time) (retryAfter int, ok bool) { + l.mu.Lock() + defer l.mu.Unlock() + if len(l.keys) >= maxPeerLimiterKeys { + clear(l.keys) + } + kb := l.keys[key] + if kb == nil { + kb = &peerBucket{} + l.keys[key] = kb + } + // Check both before spending either, so a refusal costs nothing. + probeKey, probeGlobal := *kb, l.global + if !probeKey.take(now, l.keyRate, l.keyBurst) { + return max(1, int(1/max(l.keyRate, 1e-6))), false + } + if !probeGlobal.take(now, l.globalRate, l.globalBurst) { + return max(1, int(1/max(l.globalRate, 1e-6))), false + } + *kb, l.global = probeKey, probeGlobal + return 0, true +} + +// peerLimiter returns the limiter, created with the defaults on first use: 6 +// plays per minute per key (burst 3) and 20 per minute overall (burst 6). +func (s *Server) peerLimiter() *peerPlayLimiter { + if l := s.peerPlayLimits.Load(); l != nil { + return l + } + s.peerPlayLimits.CompareAndSwap(nil, newPeerPlayLimiter(6, 3, 20, 6)) + return s.peerPlayLimits.Load() +} + +// peerQueueMetadata is the queue metadata of a peer play. Its media ids use +// "unverified_" keys on purpose: nothing that reads import metadata for media +// ids (the record publisher) recognises them. +type peerQueueMetadata struct { + ContentKey string `json:"content_key"` + ReleaseKey string `json:"release_key"` + UnverifiedIMDbID string `json:"unverified_imdb_id,omitempty"` + UnverifiedType string `json:"unverified_type,omitempty"` + UnverifiedSeason int `json:"unverified_season,omitempty"` + UnverifiedEpisode int `json:"unverified_episode,omitempty"` +} diff --git a/internal/api/stremio_peer_test.go b/internal/api/stremio_peer_test.go new file mode 100644 index 000000000..9789f3afb --- /dev/null +++ b/internal/api/stremio_peer_test.go @@ -0,0 +1,436 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/recover" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/kipsilabs/altmount/internal/auth" + "github.com/kipsilabs/altmount/internal/config" + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/peernet/service" +) + +var relA = strings.Repeat("a", 64) + +type fakePlayback struct { + mu sync.Mutex + cands []service.PeerCandidate + offers map[string]bool + playErr error + result PeerPlayResult + gate chan struct{} + candCalls int + gotSeason *int + gotEpisode *int + offerCalls atomic.Int32 + playCalls atomic.Int32 + lastReq PeerPlayRequest +} + +func (f *fakePlayback) Candidates(_ context.Context, _ string, season, episode *int) []service.PeerCandidate { + f.mu.Lock() + defer f.mu.Unlock() + f.candCalls++ + f.gotSeason, f.gotEpisode = season, episode + return f.cands +} +func (f *fakePlayback) HasOffer(_ context.Context, rel string) bool { + f.offerCalls.Add(1) + return f.offers[rel] +} +func (f *fakePlayback) Play(_ context.Context, req PeerPlayRequest) (PeerPlayResult, error) { + f.playCalls.Add(1) + f.mu.Lock() + f.lastReq = req + f.mu.Unlock() + if f.gate != nil { + <-f.gate + } + return f.result, f.playErr +} + +func TestPeerStreams_FeatureOffAddsNothingAndAsksNothing(t *testing.T) { + s := &Server{} + assert.Nil(t, s.peerCandidateStreams(context.Background(), &config.Config{}, "movie", "tt1", "http://h", "k", 0, 0)) + f := &fakePlayback{cands: []service.PeerCandidate{{ReleaseID: relA, Title: "X"}}} + s.SetPeerPlayback(f) + s.SetPeerPlayback(nil) + assert.Nil(t, s.peerCandidateStreams(context.Background(), &config.Config{}, "movie", "tt1", "http://h", "k", 0, 0)) + assert.Zero(t, f.candCalls) +} + +func TestPeerStreams_ListEntriesAreLabelledSafeAndBounded(t *testing.T) { + cands := []service.PeerCandidate{ + {ReleaseID: relA, Title: "Film.2020.1080p.WEB-DL-GRP", TotalSize: 5_000_000_000, Healthy: true}, + {ReleaseID: strings.Repeat("b", 64), Title: ` http://evil.example/x?y=../../etc`, TotalSize: 1}, + {ReleaseID: "not-hex", Title: "bad id"}, + {ReleaseID: strings.Repeat("C", 64), Title: "upper case id"}, + } + for range 20 { + cands = append(cands, service.PeerCandidate{ReleaseID: strings.Repeat("d", 64), Title: "dup"}) + } + f := &fakePlayback{cands: cands} + s := &Server{} + s.SetPeerPlayback(f) + got := s.peerCandidateStreams(context.Background(), &config.Config{}, "movie", "tt0111161", "http://host", "thekey", 0, 0) + require.GreaterOrEqual(t, len(got), 2) + assert.LessOrEqual(t, len(got), maxPeerStreams) + first := got[0] + assert.Contains(t, first["title"], peerStreamMarker) + assert.Contains(t, first["title"], "verified on play") + assert.Contains(t, first["title"], stremioNetworkHealthyLabel, "healthy on the network is said in the fixed local words") + u, err := url.Parse(first["url"].(string)) + require.NoError(t, err) + assert.Equal(t, "/stremio/thekey/peer/"+relA, u.Path, "the link names the release, never an address or a bundle id") + assert.Equal(t, "movie", u.Query().Get("type")) + assert.Equal(t, "tt0111161", u.Query().Get("id")) + // Peer text is only ever data inside a JSON string, never a URL or path part. + for _, st := range got { + pu, err := url.Parse(st["url"].(string)) + require.NoError(t, err) + assert.True(t, strings.HasPrefix(pu.Path, "/stremio/thekey/peer/")) + assert.NotContains(t, st["url"], "evil.example") + assert.NotContains(t, st["url"], "script") + assert.Regexp(t, `^[0-9a-f]{64}$`, strings.TrimPrefix(pu.Path, "/stremio/thekey/peer/")) + } + assert.Nil(t, f.gotSeason) + assert.Nil(t, f.gotEpisode, "a movie asks for the movie key") + + // A series asks with its season and episode; without them there is no key. + f.cands = nil + s.peerCandidateStreams(context.Background(), &config.Config{}, "series", "tt1", "http://host", "k", 2, 5) + require.NotNil(t, f.gotSeason) + assert.Equal(t, 2, *f.gotSeason) + assert.Equal(t, 5, *f.gotEpisode) + before := f.candCalls + assert.Nil(t, s.peerCandidateStreams(context.Background(), &config.Config{}, "series", "tt1", "http://host", "k", 0, 0)) + assert.Equal(t, before, f.candCalls, "no season and episode: no lookup") +} + +type peerHarness struct { + app *fiber.App + srv *Server + key string + f *fakePlayback +} + +func newPeerHarness(t *testing.T, f *fakePlayback) *peerHarness { + t.Helper() + db, err := database.NewDB(database.Config{DatabasePath: filepath.Join(t.TempDir(), "test.db")}) + require.NoError(t, err) + t.Cleanup(func() { db.Close() }) + userRepo := database.NewUserRepository(db.Connection(), database.DialectSQLite) + apiKey := "test-api-key" + require.NoError(t, userRepo.CreateUser(context.Background(), &database.User{UserID: "admin", Provider: "local", APIKey: &apiKey})) + cfg := &config.Config{} + cfg.Stremio.Enabled = boolPtr(true) + srv := &Server{configManager: &mockConfigManager{cfg: cfg}, userRepo: userRepo} + if f != nil { + srv.SetPeerPlayback(f) + } + app := fiber.New() + app.Use(recover.New()) + app.Get("/stremio/:key/peer/:release", srv.handleStremioPeerPlay) + return &peerHarness{app: app, srv: srv, key: auth.HashAPIKey(apiKey), f: f} +} + +func (h *peerHarness) get(t *testing.T, key, release, query string) *http.Response { + t.Helper() + resp, err := h.app.Test(httptest.NewRequest(http.MethodGet, "/stremio/"+key+"/peer/"+release+"?"+query, nil), -1) + require.NoError(t, err) + t.Cleanup(func() { resp.Body.Close() }) + return resp +} + +func TestPeerPlay_ImportsOnceAndRedirectsToTheLibraryStream(t *testing.T) { + f := &fakePlayback{offers: map[string]bool{relA: true}, result: PeerPlayResult{ + Paths: []string{"complete/Movies/R/sample.mkv", "complete/Movies/R/Film.nfo", "complete/Movies/R/Film.mkv", "complete/Movies/R/extra.mkv"}, + Sizes: []int64{10, 5, 5000, 700}}} + h := newPeerHarness(t, f) + resp := h.get(t, h.key, relA, "type=movie&id=tt0111161") + require.Equal(t, http.StatusFound, resp.StatusCode) + loc, err := url.Parse(resp.Header.Get("Location")) + require.NoError(t, err) + assert.Equal(t, "/api/files/stream", loc.Path) + assert.Equal(t, "complete/Movies/R/Film.mkv", loc.Query().Get("path"), "the main media file, not a sample or sidecar") + assert.Equal(t, h.key, loc.Query().Get("download_key")) + + req := f.lastReq + assert.Equal(t, relA, req.ReleaseID) + assert.Equal(t, "Movies", req.Category) + assert.True(t, strings.HasPrefix(req.DownloadID, stremioDownloadIDPrefix)) + assert.Contains(t, req.Metadata, `"unverified_imdb_id":"tt0111161"`, "the media id is kept for local use, marked unverified") + assert.NotContains(t, req.Metadata, `"imdb_id"`, "a peer's claim must not parse as a publishable media id") + assert.NotContains(t, req.Metadata, `"type"`) + assert.Contains(t, req.Metadata, `"content_key":"movie:tt0111161:0:0"`, "the listing still dedups on the content key") +} + +func TestPeerPlay_DifferentContentOfOneReleaseDoesNotShareAnImport(t *testing.T) { + f := &fakePlayback{offers: map[string]bool{relA: true}, gate: make(chan struct{}), + result: PeerPlayResult{Paths: []string{"c/M/R/Film.mkv"}, Sizes: []int64{9}}} + h := newPeerHarness(t, f) + h.srv.peerPlayLimits.Store(newPeerPlayLimiter(100, 100, 100, 100)) + var wg sync.WaitGroup + for _, id := range []string{"tt1", "tt2"} { + wg.Add(1) + go func() { + defer wg.Done() + h.get(t, h.key, relA, "type=movie&id="+id) + }() + } + require.Eventually(t, func() bool { return f.playCalls.Load() >= 2 }, 3*time.Second, 5*time.Millisecond, + "each content key gets its own import metadata") + close(f.gate) + wg.Wait() +} + +func TestPeerPlay_SeriesCategoryAndEpisodeSelection(t *testing.T) { + f := &fakePlayback{offers: map[string]bool{relA: true}, result: PeerPlayResult{ + Paths: []string{"c/TV/R/Show.S01E01.mkv", "c/TV/R/Show.S01E02.mkv"}, Sizes: []int64{100, 100}}} + h := newPeerHarness(t, f) + resp := h.get(t, h.key, relA, "type=series&id=tt7441658&season=1&episode=2") + require.Equal(t, http.StatusFound, resp.StatusCode) + loc, _ := url.Parse(resp.Header.Get("Location")) + assert.Equal(t, "c/TV/R/Show.S01E02.mkv", loc.Query().Get("path")) + assert.Equal(t, "TV", f.lastReq.Category) + assert.Contains(t, f.lastReq.Metadata, `"unverified_season":1`) + assert.Contains(t, f.lastReq.Metadata, `"unverified_episode":2`) +} + +func TestPeerPlay_ConcurrentRequestsShareOneImport(t *testing.T) { + f := &fakePlayback{offers: map[string]bool{relA: true}, gate: make(chan struct{}), + result: PeerPlayResult{Paths: []string{"c/M/R/Film.mkv"}, Sizes: []int64{9}}} + h := newPeerHarness(t, f) + h.srv.peerPlayLimits.Store(newPeerPlayLimiter(100, 100, 100, 100)) + var wg sync.WaitGroup + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + assert.Equal(t, http.StatusFound, h.get(t, h.key, relA, "type=movie&id=tt1").StatusCode) + }() + } + require.Eventually(t, func() bool { return f.playCalls.Load() >= 1 }, 5*time.Second, 5*time.Millisecond) + time.Sleep(100 * time.Millisecond) + close(f.gate) + wg.Wait() + assert.Equal(t, int32(1), f.playCalls.Load(), "the import ran once for all concurrent plays") +} + +func TestPeerPlay_RefusesBadIdsUnknownReleasesAndBadKeysWithoutFetching(t *testing.T) { + f := &fakePlayback{offers: map[string]bool{relA: true}} + h := newPeerHarness(t, f) + for name, tc := range map[string]struct { + key, rel string + want int + }{ + "not hex": {h.key, strings.Repeat("z", 64), http.StatusBadRequest}, + "too short": {h.key, "abcd", http.StatusBadRequest}, + "upper case": {h.key, strings.Repeat("A", 64), http.StatusBadRequest}, + "path-like": {h.key, "..%2f..%2fetc%2fpasswd", http.StatusBadRequest}, + "unknown release": {h.key, strings.Repeat("e", 64), http.StatusNotFound}, + "wrong key": {"nope", relA, http.StatusUnauthorized}, + } { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.want, h.get(t, tc.key, tc.rel, "type=movie&id=tt1").StatusCode) + }) + } + assert.Zero(t, f.playCalls.Load(), "none of these may start a fetch") +} + +func TestPeerPlay_FeatureOffOrStremioOffIsNotFound(t *testing.T) { + h := newPeerHarness(t, nil) + assert.Equal(t, http.StatusNotFound, h.get(t, h.key, relA, "type=movie&id=tt1").StatusCode) + f := &fakePlayback{offers: map[string]bool{relA: true}} + h2 := newPeerHarness(t, f) + h2.srv.configManager.GetConfig().Stremio.Enabled = boolPtr(false) + assert.Equal(t, http.StatusNotFound, h2.get(t, h2.key, relA, "type=movie&id=tt1").StatusCode) + assert.Zero(t, f.offerCalls.Load()) +} + +func TestPeerPlay_FailureLooksLikeAFailedNZBAndLeaksNothing(t *testing.T) { + f := &fakePlayback{offers: map[string]bool{relA: true}, playErr: assertErr("fetch 10.0.0.5:8470 failed: /var/secret/path")} + h := newPeerHarness(t, f) + resp := h.get(t, h.key, relA, "type=movie&id=tt1") + assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode) + body := readAll(t, resp) + assert.NotContains(t, body, "10.0.0.5") + assert.NotContains(t, body, "/var/secret") + + f2 := &fakePlayback{offers: map[string]bool{relA: true}, result: PeerPlayResult{Paths: []string{"c/M/R/Film.nfo"}, Sizes: []int64{1}}} + h2 := newPeerHarness(t, f2) + assert.Equal(t, http.StatusServiceUnavailable, h2.get(t, h2.key, relA, "type=movie&id=tt1").StatusCode, "no media file in the release") +} + +func TestPeerPlay_RateLimitedPerKeyAndGlobally(t *testing.T) { + f := &fakePlayback{offers: map[string]bool{relA: true}, result: PeerPlayResult{Paths: []string{"c/M/R/Film.mkv"}, Sizes: []int64{9}}} + h := newPeerHarness(t, f) + h.srv.peerPlayLimits.Store(newPeerPlayLimiter(2, 2, 100, 100)) + codes := []int{} + for range 4 { + codes = append(codes, h.get(t, h.key, relA, "type=movie&id=tt1").StatusCode) + } + assert.Equal(t, []int{302, 302, 429, 429}, codes, "burst of 2 per key, then refused") + + g := newPeerHarness(t, &fakePlayback{offers: map[string]bool{relA: true}, result: f.result}) + g.srv.peerPlayLimits.Store(newPeerPlayLimiter(100, 100, 1, 1)) + assert.Equal(t, 302, g.get(t, g.key, relA, "type=movie&id=tt1").StatusCode) + assert.Equal(t, 429, g.get(t, g.key, relA, "type=movie&id=tt1").StatusCode, "the global budget holds whatever the key") + assert.NotEmpty(t, g.get(t, g.key, relA, "type=movie&id=tt1").Header.Get("Retry-After")) +} + +type assertErr string + +func (e assertErr) Error() string { return string(e) } + +func readAll(t *testing.T, resp *http.Response) string { + t.Helper() + var sb strings.Builder + buf := make([]byte, 4096) + for { + n, err := resp.Body.Read(buf) + sb.Write(buf[:n]) + if err != nil { + break + } + } + return sb.String() +} + +func streamList(t *testing.T, h *peerHarness, path string) (string, int) { + t.Helper() + h.app.Get("/stremio/:key/stream/:type/:id.json", h.srv.handleStremioAddonStream) + resp, err := h.app.Test(httptest.NewRequest(http.MethodGet, "/stremio/"+h.key+"/stream/"+path, nil), -1) + require.NoError(t, err) + t.Cleanup(func() { resp.Body.Close() }) + return readAll(t, resp), resp.StatusCode +} + +func TestStreamList_PeerCandidatesAddedOnlyWithTheFeature(t *testing.T) { + off := newPeerHarness(t, nil) + offBody, code := streamList(t, off, "movie/tt0111161.json") + require.Equal(t, 200, code) + + // Feature on, nothing to offer: the response is byte-identical to off. + empty := newPeerHarness(t, &fakePlayback{}) + emptyBody, _ := streamList(t, empty, "movie/tt0111161.json") + assert.Equal(t, offBody, emptyBody) + + // With a candidate the list carries it, and keeps working when the indexer + // search has nothing. + f := &fakePlayback{cands: []service.PeerCandidate{{ReleaseID: relA, Title: "Film.2020.1080p-GRP", TotalSize: 4_000_000_000}}} + on := newPeerHarness(t, f) + body, _ := streamList(t, on, "movie/tt0111161.json") + assert.Contains(t, body, peerStreamMarker) + assert.Contains(t, body, "/stremio/"+on.key+"/peer/"+relA) + assert.Contains(t, body, "Film.2020.1080p-GRP") +} + +// funcPlayback is a PeerPlayback whose candidate lookup is a function. +type funcPlayback struct { + fakePlayback + lookup func(ctx context.Context) []service.PeerCandidate +} + +func (f *funcPlayback) Candidates(ctx context.Context, _ string, _, _ *int) []service.PeerCandidate { + return f.lookup(ctx) +} + +func peerServer(f PeerPlayback) *Server { + srv := &Server{} + srv.SetPeerPlayback(f) + return srv +} + +func startPeer(srv *Server) func() []fiber.Map { + return srv.startPeerCandidates(context.Background(), &config.Config{}, "movie", "tt0111161", "http://h", "k", 0, 0) +} + +// The peer lookup runs next to the indexer search: a lookup as slow as the +// search adds nothing to the listing. +func TestPeerStreams_LookupRunsConcurrentlyWithTheSearch(t *testing.T) { + const d = 300 * time.Millisecond + srv := peerServer(&funcPlayback{lookup: func(context.Context) []service.PeerCandidate { + time.Sleep(d) + return []service.PeerCandidate{{ReleaseID: relA, Title: "Film.2020.1080p-GRP"}} + }}) + start := time.Now() + wait := startPeer(srv) + time.Sleep(d) // the indexer search + got := wait() + elapsed := time.Since(start) + require.Len(t, got, 1, "the lookup finished with the search and is merged") + assert.Less(t, elapsed, d+d/2, "serial would take twice as long") +} + +// A lookup that hangs never delays the response by more than the grace, and +// the listing carries no peer stream. +func TestPeerStreams_SlowLookupIsBoundedAndFailsOpen(t *testing.T) { + cancelled := make(chan struct{}) + srv := peerServer(&funcPlayback{lookup: func(ctx context.Context) []service.PeerCandidate { + <-ctx.Done() + close(cancelled) + return nil + }}) + wait := startPeer(srv) + start := time.Now() + got := wait() // the search ended at once + assert.Empty(t, got) + assert.Less(t, time.Since(start), peerCandidateGrace+200*time.Millisecond) + select { + case <-cancelled: + case <-time.After(2 * time.Second): + t.Fatal("the abandoned lookup was not cancelled") + } +} + +func TestPeerStreams_PanickingLookupFailsOpen(t *testing.T) { + srv := peerServer(&funcPlayback{lookup: func(context.Context) []service.PeerCandidate { panic("boom") }}) + assert.Empty(t, startPeer(srv)()) +} + +func TestPeerStreams_FeatureOffStartsNothing(t *testing.T) { + assert.Nil(t, startPeer(&Server{})()) +} + +// With no peer streams the merged listing is byte for byte what a listing +// without the peer network produced: library streams, then indexer entries. +func TestStreamList_MergeWithoutPeerStreamsIsByteIdenticalToTheFormerOutput(t *testing.T) { + library := []fiber.Map{{"name": "L", "title": "lib", "url": "http://h/api/files/stream?path=a"}} + entries := []stremioStreamEntry{{Name: "N", Title: "T", URL: "http://h/u", BehaviorHints: stremioStreamBehaviorHints{Service: "altmount", Filename: "f"}}} + + got, err := json.Marshal(fiber.Map{"streams": assembleStremioStreams(library, nil, entries)}) + require.NoError(t, err) + const golden = `{"streams":[{"name":"L","title":"lib","url":"http://h/api/files/stream?path=a"},` + + `{"behaviorHints":{"service":"altmount","cached":false,"filename":"f"},"name":"N","title":"T","url":"http://h/u"}]}` + assert.JSONEq(t, golden, string(got)) + // Key order of fiber.Map marshalling is alphabetical and stable; compare the + // raw bytes against the former construction as well. + former := make([]fiber.Map, 0, 2) + former = append(former, library...) + for _, e := range entries { + former = append(former, fiber.Map{"name": e.Name, "title": e.Title, "url": e.URL, "behaviorHints": e.BehaviorHints}) + } + want, err := json.Marshal(fiber.Map{"streams": former}) + require.NoError(t, err) + assert.Equal(t, string(want), string(got)) + + // Peer streams slot between the two, in order. + merged := assembleStremioStreams(library, []fiber.Map{{"name": "P"}}, entries) + require.Len(t, merged, 3) + assert.Equal(t, "P", merged[1]["name"]) +} diff --git a/internal/api/types.go b/internal/api/types.go index 78ca496de..acf14cb47 100644 --- a/internal/api/types.go +++ b/internal/api/types.go @@ -672,6 +672,10 @@ type QueueItemResponse struct { Percentage *int `json:"percentage,omitempty"` // Progress percentage (0-100), only for items being processed Stage string `json:"stage,omitempty"` // Progress stage (e.g. "Validating segments") StoragePath *string `json:"storage_path,omitempty"` // Internal FUSE mount path (populated after completion) + // PeerOrigin: the import came from a peer-network bundle. PeerFullCheckPending: + // its full check has not passed yet (it is not announced to the ARRs). + PeerOrigin bool `json:"peer_origin"` + PeerFullCheckPending bool `json:"peer_full_check_pending"` } // QueueStatsResponse represents queue statistics in API responses @@ -727,25 +731,27 @@ type QueueHistoricalStatsResponse struct { // HealthItemResponse represents a health record in API responses type HealthItemResponse struct { - CheckProgress *health.CheckProgress `json:"check_progress,omitempty"` - ID int64 `json:"id"` - FilePath string `json:"file_path"` - LibraryPath *string `json:"library_path,omitempty"` - Status database.HealthStatus `json:"status"` - LastChecked *time.Time `json:"last_checked"` - LastError *string `json:"last_error"` - RetryCount int `json:"retry_count"` - MaxRetries int `json:"max_retries"` - SourceNzbPath *string `json:"source_nzb_path"` - ErrorDetails *string `json:"error_details"` - RepairRetryCount int `json:"repair_retry_count"` - MaxRepairRetries int `json:"max_repair_retries"` - Indexer *string `json:"indexer,omitempty"` // Added indexer - Metadata *string `json:"metadata,omitempty"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - ScheduledCheckAt *time.Time `json:"scheduled_check_at,omitempty"` - Priority database.HealthPriority `json:"priority"` + PeerOrigin bool `json:"peer_origin"` + PeerFullCheckPending bool `json:"peer_full_check_pending"` + CheckProgress *health.CheckProgress `json:"check_progress,omitempty"` + ID int64 `json:"id"` + FilePath string `json:"file_path"` + LibraryPath *string `json:"library_path,omitempty"` + Status database.HealthStatus `json:"status"` + LastChecked *time.Time `json:"last_checked"` + LastError *string `json:"last_error"` + RetryCount int `json:"retry_count"` + MaxRetries int `json:"max_retries"` + SourceNzbPath *string `json:"source_nzb_path"` + ErrorDetails *string `json:"error_details"` + RepairRetryCount int `json:"repair_retry_count"` + MaxRepairRetries int `json:"max_repair_retries"` + Indexer *string `json:"indexer,omitempty"` // Added indexer + Metadata *string `json:"metadata,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ScheduledCheckAt *time.Time `json:"scheduled_check_at,omitempty"` + Priority database.HealthPriority `json:"priority"` // Failure masking fields StreamingFailureCount int `json:"streaming_failure_count"` IsMasked bool `json:"is_masked"` diff --git a/internal/config/peernet.go b/internal/config/peernet.go index 45b03d858..b142d6f1d 100644 --- a/internal/config/peernet.go +++ b/internal/config/peernet.go @@ -69,6 +69,12 @@ type PeernetOpenNetworkConfig struct { Level PeernetLevel `yaml:"level" mapstructure:"level" json:"level"` // Peers are host:port addresses (no scheme, userinfo or path). Peers []string `yaml:"peers" mapstructure:"peers" json:"peers"` + // FastPath lets layouts offered by open-network peers replace archive + // analysis when the user imports an NZB. Off by default: such a layout is + // verified by sampling only. Private networks do not need it (their + // offers use the fast path by default); direct imports (Stremio peer play) + // are unaffected. + FastPath bool `yaml:"fast_path" mapstructure:"fast_path" json:"fast_path"` } // PeernetSharingConfig holds publish-time exclusions. diff --git a/internal/config/peernet_test.go b/internal/config/peernet_test.go index 69088ebec..965da1ffa 100644 --- a/internal/config/peernet_test.go +++ b/internal/config/peernet_test.go @@ -21,6 +21,7 @@ func TestPeernetDefaults(t *testing.T) { assert.Empty(t, c.Peernet.AdvertiseAddress) assert.Equal(t, PeernetLevelOff, c.Peernet.OpenNetwork.Level) assert.Empty(t, c.Peernet.OpenNetwork.Peers) + assert.False(t, c.Peernet.OpenNetwork.FastPath, "open-network layouts never replace archive analysis unless the user opts in") assert.Empty(t, c.Peernet.Sharing.ExcludedCategories) assert.Empty(t, c.Peernet.Sharing.ExcludedIndexers) assert.Equal(t, 3, c.Peernet.MinCorroboration) diff --git a/internal/database/migrations/postgres/057_add_peernet_release_scheme.sql b/internal/database/migrations/postgres/057_add_peernet_release_scheme.sql new file mode 100644 index 000000000..a660478e0 --- /dev/null +++ b/internal/database/migrations/postgres/057_add_peernet_release_scheme.sql @@ -0,0 +1,24 @@ +-- +goose Up +-- How each indexed file's release_id was derived: 0 none, 1 "nzb" (hash of the +-- release's NZB segment list), 2 "files" (derived from the File IDs of the +-- files of one import, for metadata that predates the NZB store). Existing +-- rows with a release_id came from a store, so they are scheme 1. +ALTER TABLE peernet_content_index ADD COLUMN release_scheme INTEGER NOT NULL DEFAULT 0; +-- A local-only grouping key (hash of the metadata's source NZB path) that +-- says which indexed files came from one import. It is never transmitted. +ALTER TABLE peernet_content_index ADD COLUMN group_key TEXT NOT NULL DEFAULT ''; +UPDATE peernet_content_index SET release_scheme = 1 WHERE release_id <> ''; +CREATE INDEX idx_peernet_content_group ON peernet_content_index(group_key) WHERE group_key <> ''; + +-- Which scheme a stored record's release_id uses (1 nzb, 2 files). Records +-- stored before this migration are scheme 1. +ALTER TABLE peernet_records ADD COLUMN release_scheme INTEGER NOT NULL DEFAULT 1; +-- Health evidence is matched by file ID across release IDs. +CREATE INDEX idx_peernet_records_file ON peernet_records(network_id, file_id, kind); + +-- +goose Down +DROP INDEX IF EXISTS idx_peernet_records_file; +ALTER TABLE peernet_records DROP COLUMN release_scheme; +DROP INDEX IF EXISTS idx_peernet_content_group; +ALTER TABLE peernet_content_index DROP COLUMN group_key; +ALTER TABLE peernet_content_index DROP COLUMN release_scheme; diff --git a/internal/database/migrations/postgres/058_add_peernet_publish_bundle.sql b/internal/database/migrations/postgres/058_add_peernet_publish_bundle.sql new file mode 100644 index 000000000..a98967488 --- /dev/null +++ b/internal/database/migrations/postgres/058_add_peernet_publish_bundle.sql @@ -0,0 +1,16 @@ +-- +goose Up +-- The Bundle ID this node announced in a release record, and the exposure +-- level the record was published at. A record published at level 3 carries a +-- Bundle ID (or none, when the release cannot be shared as a bundle); at level +-- 2 it never does. Comparing bundle_level with the network's current level is +-- how a level change is noticed and the record superseded. bundle_id is also +-- the key by which a bundle request is matched to an announcement: a node +-- serves only a Bundle ID it announced itself. +ALTER TABLE peernet_publish_state ADD COLUMN bundle_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE peernet_publish_state ADD COLUMN bundle_level INTEGER NOT NULL DEFAULT 0; +CREATE INDEX idx_peernet_publish_state_bundle ON peernet_publish_state(network_id, bundle_id) WHERE bundle_id <> ''; + +-- +goose Down +DROP INDEX IF EXISTS idx_peernet_publish_state_bundle; +ALTER TABLE peernet_publish_state DROP COLUMN bundle_level; +ALTER TABLE peernet_publish_state DROP COLUMN bundle_id; diff --git a/internal/database/migrations/postgres/059_add_peernet_record_bundle.sql b/internal/database/migrations/postgres/059_add_peernet_record_bundle.sql new file mode 100644 index 000000000..47b23c6ee --- /dev/null +++ b/internal/database/migrations/postgres/059_add_peernet_record_bundle.sql @@ -0,0 +1,8 @@ +-- +goose Up +-- The Bundle ID a stored release record announces (empty: the publisher serves +-- no bundle). It comes from the signed record. Offers for a release are found +-- through the existing (network, release, kind) index. +ALTER TABLE peernet_records ADD COLUMN bundle_id TEXT NOT NULL DEFAULT ''; + +-- +goose Down +ALTER TABLE peernet_records DROP COLUMN bundle_id; diff --git a/internal/database/migrations/postgres/060_add_peernet_bundle_imports.sql b/internal/database/migrations/postgres/060_add_peernet_bundle_imports.sql new file mode 100644 index 000000000..6e6bdcc3e --- /dev/null +++ b/internal/database/migrations/postgres/060_add_peernet_bundle_imports.sql @@ -0,0 +1,33 @@ +-- +goose Up +-- Imports that came from a peer bundle, one row per virtual file. A bundle +-- import is playable at once but is held back from the ARRs and from the peer +-- network until a full health check of every file passed (state "pending" +-- becomes "passed"); a failed check marks it "failed". queue_id groups the +-- files of one import (0: none). arr_owed says the ARRs must be told when the +-- last file passes (the import was not suppressed by its caller); +-- arr_notified is set by the single winner of that notification. network_id and +-- publisher_id name the origin, kept only so a bad bundle can be reported; +-- they are never shown as a link. Times are Unix seconds. +CREATE TABLE peernet_bundle_imports ( + file_path TEXT NOT NULL PRIMARY KEY, + file_id TEXT NOT NULL, + release_id TEXT NOT NULL, + network_id TEXT NOT NULL, + publisher_id TEXT NOT NULL, + queue_id BIGINT NOT NULL DEFAULT 0, + state TEXT NOT NULL DEFAULT 'pending', + arr_owed INTEGER NOT NULL DEFAULT 0, + arr_notified INTEGER NOT NULL DEFAULT 0, + category TEXT NOT NULL DEFAULT '', + target_path TEXT NOT NULL DEFAULT '', + resulting_path TEXT NOT NULL DEFAULT '', + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL +); +CREATE INDEX idx_peernet_bundle_imports_queue ON peernet_bundle_imports(queue_id) WHERE queue_id <> 0; +CREATE INDEX idx_peernet_bundle_imports_release ON peernet_bundle_imports(release_id); + +-- +goose Down +DROP INDEX IF EXISTS idx_peernet_bundle_imports_release; +DROP INDEX IF EXISTS idx_peernet_bundle_imports_queue; +DROP TABLE IF EXISTS peernet_bundle_imports; diff --git a/internal/database/migrations/postgres/061_add_peernet_password_origin.sql b/internal/database/migrations/postgres/061_add_peernet_password_origin.sql new file mode 100644 index 000000000..b9f4ae9c0 --- /dev/null +++ b/internal/database/migrations/postgres/061_add_peernet_password_origin.sql @@ -0,0 +1,15 @@ +-- +goose Up +-- Files whose archive password came with the NZB's own content (a meta +-- password present in the NZB as received, not one the user typed, an ARR +-- supplied or the NZB name spelled). A row exists only for such files; a file +-- with no row is "not from the NZB", which is also what every file imported +-- before this table existed is. It is keyed by File ID (derived from the +-- file's segments), so a rename or move of the file changes nothing. The +-- password itself is never stored. Times are Unix seconds. +CREATE TABLE peernet_password_origin ( + file_id TEXT NOT NULL PRIMARY KEY, + recorded_at BIGINT NOT NULL +); + +-- +goose Down +DROP TABLE IF EXISTS peernet_password_origin; diff --git a/internal/database/migrations/postgres/062_add_import_queue_peer_hold.sql b/internal/database/migrations/postgres/062_add_import_queue_peer_hold.sql new file mode 100644 index 000000000..17a171c2d --- /dev/null +++ b/internal/database/migrations/postgres/062_add_import_queue_peer_hold.sql @@ -0,0 +1,20 @@ +-- +goose Up +-- peer_hold marks a queue item whose files came from a peer bundle and still +-- owe their full health check (1), as opposed to skip_arr_notification, which +-- only ever records what the user asked for. The SABnzbd-compatible API shows a +-- held item in its queue (never in its history) until the check ends: a pass +-- clears the mark, a failure turns the item into a failed one. 0 is every row +-- that never came from the peer network. +ALTER TABLE import_queue ADD COLUMN peer_hold SMALLINT NOT NULL DEFAULT 0; +CREATE INDEX idx_import_queue_peer_hold ON import_queue(peer_hold) WHERE peer_hold <> 0; + +-- committed says a bundle import finished writing all its files: a row is +-- inserted uncommitted (0) before the first file is written and committed last, +-- so a crash in between is told apart at start-up and rolled back whole. Rows +-- that existed before this column are complete ones (default 1). +ALTER TABLE peernet_bundle_imports ADD COLUMN committed SMALLINT NOT NULL DEFAULT 1; + +-- +goose Down +ALTER TABLE peernet_bundle_imports DROP COLUMN committed; +DROP INDEX IF EXISTS idx_import_queue_peer_hold; +ALTER TABLE import_queue DROP COLUMN peer_hold; diff --git a/internal/database/migrations/sqlite/057_add_peernet_release_scheme.sql b/internal/database/migrations/sqlite/057_add_peernet_release_scheme.sql new file mode 100644 index 000000000..a660478e0 --- /dev/null +++ b/internal/database/migrations/sqlite/057_add_peernet_release_scheme.sql @@ -0,0 +1,24 @@ +-- +goose Up +-- How each indexed file's release_id was derived: 0 none, 1 "nzb" (hash of the +-- release's NZB segment list), 2 "files" (derived from the File IDs of the +-- files of one import, for metadata that predates the NZB store). Existing +-- rows with a release_id came from a store, so they are scheme 1. +ALTER TABLE peernet_content_index ADD COLUMN release_scheme INTEGER NOT NULL DEFAULT 0; +-- A local-only grouping key (hash of the metadata's source NZB path) that +-- says which indexed files came from one import. It is never transmitted. +ALTER TABLE peernet_content_index ADD COLUMN group_key TEXT NOT NULL DEFAULT ''; +UPDATE peernet_content_index SET release_scheme = 1 WHERE release_id <> ''; +CREATE INDEX idx_peernet_content_group ON peernet_content_index(group_key) WHERE group_key <> ''; + +-- Which scheme a stored record's release_id uses (1 nzb, 2 files). Records +-- stored before this migration are scheme 1. +ALTER TABLE peernet_records ADD COLUMN release_scheme INTEGER NOT NULL DEFAULT 1; +-- Health evidence is matched by file ID across release IDs. +CREATE INDEX idx_peernet_records_file ON peernet_records(network_id, file_id, kind); + +-- +goose Down +DROP INDEX IF EXISTS idx_peernet_records_file; +ALTER TABLE peernet_records DROP COLUMN release_scheme; +DROP INDEX IF EXISTS idx_peernet_content_group; +ALTER TABLE peernet_content_index DROP COLUMN group_key; +ALTER TABLE peernet_content_index DROP COLUMN release_scheme; diff --git a/internal/database/migrations/sqlite/058_add_peernet_publish_bundle.sql b/internal/database/migrations/sqlite/058_add_peernet_publish_bundle.sql new file mode 100644 index 000000000..a98967488 --- /dev/null +++ b/internal/database/migrations/sqlite/058_add_peernet_publish_bundle.sql @@ -0,0 +1,16 @@ +-- +goose Up +-- The Bundle ID this node announced in a release record, and the exposure +-- level the record was published at. A record published at level 3 carries a +-- Bundle ID (or none, when the release cannot be shared as a bundle); at level +-- 2 it never does. Comparing bundle_level with the network's current level is +-- how a level change is noticed and the record superseded. bundle_id is also +-- the key by which a bundle request is matched to an announcement: a node +-- serves only a Bundle ID it announced itself. +ALTER TABLE peernet_publish_state ADD COLUMN bundle_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE peernet_publish_state ADD COLUMN bundle_level INTEGER NOT NULL DEFAULT 0; +CREATE INDEX idx_peernet_publish_state_bundle ON peernet_publish_state(network_id, bundle_id) WHERE bundle_id <> ''; + +-- +goose Down +DROP INDEX IF EXISTS idx_peernet_publish_state_bundle; +ALTER TABLE peernet_publish_state DROP COLUMN bundle_level; +ALTER TABLE peernet_publish_state DROP COLUMN bundle_id; diff --git a/internal/database/migrations/sqlite/059_add_peernet_record_bundle.sql b/internal/database/migrations/sqlite/059_add_peernet_record_bundle.sql new file mode 100644 index 000000000..47b23c6ee --- /dev/null +++ b/internal/database/migrations/sqlite/059_add_peernet_record_bundle.sql @@ -0,0 +1,8 @@ +-- +goose Up +-- The Bundle ID a stored release record announces (empty: the publisher serves +-- no bundle). It comes from the signed record. Offers for a release are found +-- through the existing (network, release, kind) index. +ALTER TABLE peernet_records ADD COLUMN bundle_id TEXT NOT NULL DEFAULT ''; + +-- +goose Down +ALTER TABLE peernet_records DROP COLUMN bundle_id; diff --git a/internal/database/migrations/sqlite/060_add_peernet_bundle_imports.sql b/internal/database/migrations/sqlite/060_add_peernet_bundle_imports.sql new file mode 100644 index 000000000..6e6bdcc3e --- /dev/null +++ b/internal/database/migrations/sqlite/060_add_peernet_bundle_imports.sql @@ -0,0 +1,33 @@ +-- +goose Up +-- Imports that came from a peer bundle, one row per virtual file. A bundle +-- import is playable at once but is held back from the ARRs and from the peer +-- network until a full health check of every file passed (state "pending" +-- becomes "passed"); a failed check marks it "failed". queue_id groups the +-- files of one import (0: none). arr_owed says the ARRs must be told when the +-- last file passes (the import was not suppressed by its caller); +-- arr_notified is set by the single winner of that notification. network_id and +-- publisher_id name the origin, kept only so a bad bundle can be reported; +-- they are never shown as a link. Times are Unix seconds. +CREATE TABLE peernet_bundle_imports ( + file_path TEXT NOT NULL PRIMARY KEY, + file_id TEXT NOT NULL, + release_id TEXT NOT NULL, + network_id TEXT NOT NULL, + publisher_id TEXT NOT NULL, + queue_id BIGINT NOT NULL DEFAULT 0, + state TEXT NOT NULL DEFAULT 'pending', + arr_owed INTEGER NOT NULL DEFAULT 0, + arr_notified INTEGER NOT NULL DEFAULT 0, + category TEXT NOT NULL DEFAULT '', + target_path TEXT NOT NULL DEFAULT '', + resulting_path TEXT NOT NULL DEFAULT '', + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL +); +CREATE INDEX idx_peernet_bundle_imports_queue ON peernet_bundle_imports(queue_id) WHERE queue_id <> 0; +CREATE INDEX idx_peernet_bundle_imports_release ON peernet_bundle_imports(release_id); + +-- +goose Down +DROP INDEX IF EXISTS idx_peernet_bundle_imports_release; +DROP INDEX IF EXISTS idx_peernet_bundle_imports_queue; +DROP TABLE IF EXISTS peernet_bundle_imports; diff --git a/internal/database/migrations/sqlite/061_add_peernet_password_origin.sql b/internal/database/migrations/sqlite/061_add_peernet_password_origin.sql new file mode 100644 index 000000000..b9f4ae9c0 --- /dev/null +++ b/internal/database/migrations/sqlite/061_add_peernet_password_origin.sql @@ -0,0 +1,15 @@ +-- +goose Up +-- Files whose archive password came with the NZB's own content (a meta +-- password present in the NZB as received, not one the user typed, an ARR +-- supplied or the NZB name spelled). A row exists only for such files; a file +-- with no row is "not from the NZB", which is also what every file imported +-- before this table existed is. It is keyed by File ID (derived from the +-- file's segments), so a rename or move of the file changes nothing. The +-- password itself is never stored. Times are Unix seconds. +CREATE TABLE peernet_password_origin ( + file_id TEXT NOT NULL PRIMARY KEY, + recorded_at BIGINT NOT NULL +); + +-- +goose Down +DROP TABLE IF EXISTS peernet_password_origin; diff --git a/internal/database/migrations/sqlite/062_add_import_queue_peer_hold.sql b/internal/database/migrations/sqlite/062_add_import_queue_peer_hold.sql new file mode 100644 index 000000000..738a34dc4 --- /dev/null +++ b/internal/database/migrations/sqlite/062_add_import_queue_peer_hold.sql @@ -0,0 +1,20 @@ +-- +goose Up +-- peer_hold marks a queue item whose files came from a peer bundle and still +-- owe their full health check (1), as opposed to skip_arr_notification, which +-- only ever records what the user asked for. The SABnzbd-compatible API shows a +-- held item in its queue (never in its history) until the check ends: a pass +-- clears the mark, a failure turns the item into a failed one. 0 is every row +-- that never came from the peer network. +ALTER TABLE import_queue ADD COLUMN peer_hold INTEGER NOT NULL DEFAULT 0; +CREATE INDEX idx_import_queue_peer_hold ON import_queue(peer_hold) WHERE peer_hold <> 0; + +-- committed says a bundle import finished writing all its files: a row is +-- inserted uncommitted (0) before the first file is written and committed last, +-- so a crash in between is told apart at start-up and rolled back whole. Rows +-- that existed before this column are complete ones (default 1). +ALTER TABLE peernet_bundle_imports ADD COLUMN committed INTEGER NOT NULL DEFAULT 1; + +-- +goose Down +ALTER TABLE peernet_bundle_imports DROP COLUMN committed; +DROP INDEX IF EXISTS idx_import_queue_peer_hold; +ALTER TABLE import_queue DROP COLUMN peer_hold; diff --git a/internal/database/models.go b/internal/database/models.go index 2c8e1b116..f6856310c 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -1,6 +1,8 @@ package database import ( + "path/filepath" + "strings" "time" ) @@ -55,6 +57,20 @@ type ImportQueueItem struct { SkipArrNotification bool `db:"skip_arr_notification"` SkipPostImportLinks bool `db:"skip_post_import_links"` Indexer *string `db:"indexer"` + // PeerHold is set while the item's files (from a peer bundle) owe their + // full health check. Only the active-queue list and the single-item lookups + // fill it; every other read leaves it false. + PeerHold bool `db:"-"` +} + +// PeerImportNzbPrefix starts the nzb_path of the queue row of a direct peer +// import. Such a release came from a peer's bundle, not from an NZB: there is +// no file behind the path, and the row cannot be retried, requeued or downloaded. +const PeerImportNzbPrefix = "peernet-bundle/" + +// IsPeerImport reports whether the queue item records a direct peer import. +func (i *ImportQueueItem) IsPeerImport() bool { + return i != nil && strings.HasPrefix(filepath.ToSlash(i.NzbPath), PeerImportNzbPrefix) } // BulkOperationResult represents the result of a bulk queue operation @@ -69,11 +85,11 @@ type BulkOperationResult struct { // QueueStats represents statistics about the import queue type QueueStats struct { - ID int64 `db:"id"` - TotalQueued int `db:"total_queued"` - TotalProcessing int `db:"total_processing"` - TotalCompleted int `db:"total_completed"` - TotalFailed int `db:"total_failed"` + ID int64 `db:"id"` + TotalQueued int `db:"total_queued"` + TotalProcessing int `db:"total_processing"` + TotalCompleted int `db:"total_completed"` + TotalFailed int `db:"total_failed"` // TotalWaitingRepair counts imports parked for a PAR2 repair. Counted // live; queue_stats has no column for it. TotalWaitingRepair int `db:"-"` @@ -189,18 +205,18 @@ type ImportHourlyStat struct { // ImportHistory represents a persistent record of a single imported file type ImportHistory struct { - ID int64 `db:"id"` - DownloadID *string `db:"download_id"` - NzbID *int64 `db:"nzb_id"` // Nullable if queue item deleted - NzbName string `db:"nzb_name"` - FileName string `db:"file_name"` - FileSize int64 `db:"file_size"` - VirtualPath string `db:"virtual_path"` - LibraryPath *string `db:"library_path"` // Added to show final location from file_health - Category *string `db:"category"` - Metadata *string `db:"metadata"` - Indexer *string `db:"indexer"` - CompletedAt time.Time `db:"completed_at"` + ID int64 `db:"id"` + DownloadID *string `db:"download_id"` + NzbID *int64 `db:"nzb_id"` // Nullable if queue item deleted + NzbName string `db:"nzb_name"` + FileName string `db:"file_name"` + FileSize int64 `db:"file_size"` + VirtualPath string `db:"virtual_path"` + LibraryPath *string `db:"library_path"` // Added to show final location from file_health + Category *string `db:"category"` + Metadata *string `db:"metadata"` + Indexer *string `db:"indexer"` + CompletedAt time.Time `db:"completed_at"` } // ImportMigrationStatus represents the status of a migration item diff --git a/internal/database/peernet_bundle_import_repository.go b/internal/database/peernet_bundle_import_repository.go new file mode 100644 index 000000000..e93b625e6 --- /dev/null +++ b/internal/database/peernet_bundle_import_repository.go @@ -0,0 +1,515 @@ +package database + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" +) + +// Bundle import states. +const ( + PeernetBundleStatePending = "pending" + PeernetBundleStatePassed = "passed" + PeernetBundleStateFailed = "failed" +) + +// PeernetBundleImport is one virtual file that came from a peer bundle. +type PeernetBundleImport struct { + FilePath string + FileID string + ReleaseID string + NetworkID string + PublisherID string + QueueID int64 + State string + ArrOwed bool + ArrNotified bool + Category string + TargetPath string + ResultingPath string + // Committed is set once every file of the import was written; a row + // that is not committed belongs to a write that never finished. + Committed bool +} + +// PeernetBundleImportRepository stores the peer-bundle import state. +type PeernetBundleImportRepository struct{ db *dialectAwareDB } + +// NewPeernetBundleImportRepository creates a repository over conn. +func NewPeernetBundleImportRepository(conn *sql.DB, d Dialect) *PeernetBundleImportRepository { + return &PeernetBundleImportRepository{db: newDialectAwareDB(conn, d)} +} + +const bundleImportCols = `file_path, file_id, release_id, network_id, publisher_id, queue_id, state, arr_owed, arr_notified, category, target_path, resulting_path, committed` + +// maxBundleImportBatch bounds an IN list. +const maxBundleImportBatch = 500 + +func b2i(b bool) int { + if b { + return 1 + } + return 0 +} + +func scanBundleImport(sc interface{ Scan(...any) error }) (PeernetBundleImport, error) { + var b PeernetBundleImport + var owed, notified, committed int + err := sc.Scan(&b.FilePath, &b.FileID, &b.ReleaseID, &b.NetworkID, &b.PublisherID, &b.QueueID, &b.State, + &owed, ¬ified, &b.Category, &b.TargetPath, &b.ResultingPath, &committed) + b.ArrOwed, b.ArrNotified, b.Committed = owed != 0, notified != 0, committed != 0 + return b, err +} + +// Upsert stores the rows as pending, replacing any row of the same path. The +// path is slash-normalised like every other virtual path. +func (r *PeernetBundleImportRepository) Upsert(ctx context.Context, rows []PeernetBundleImport) error { + now := time.Now().Unix() + for _, row := range rows { + p := normalizeHealthPath(row.FilePath) + if p == "" { + return errors.New("peernet bundle import: empty file path") + } + _, err := r.db.ExecContext(ctx, `INSERT INTO peernet_bundle_imports + (file_path, file_id, release_id, network_id, publisher_id, queue_id, state, arr_owed, arr_notified, category, target_path, resulting_path, committed, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, 'pending', ?, 0, ?, ?, ?, 0, ?, ?) + ON CONFLICT(file_path) DO UPDATE SET file_id = excluded.file_id, release_id = excluded.release_id, + network_id = excluded.network_id, publisher_id = excluded.publisher_id, queue_id = excluded.queue_id, + state = 'pending', arr_owed = excluded.arr_owed, arr_notified = 0, category = excluded.category, + target_path = excluded.target_path, resulting_path = excluded.resulting_path, committed = 0, updated_at = excluded.updated_at`, + p, row.FileID, row.ReleaseID, row.NetworkID, row.PublisherID, row.QueueID, b2i(row.ArrOwed), + row.Category, row.TargetPath, row.ResultingPath, now, now) + if err != nil { + return fmt.Errorf("peernet bundle import: upsert: %w", err) + } + } + return nil +} + +// Get returns the row of one path. +func (r *PeernetBundleImportRepository) Get(ctx context.Context, filePath string) (PeernetBundleImport, bool, error) { + row, err := scanBundleImport(r.db.QueryRowContext(ctx, + `SELECT `+bundleImportCols+` FROM peernet_bundle_imports WHERE file_path = ?`, normalizeHealthPath(filePath))) + if errors.Is(err, sql.ErrNoRows) { + return PeernetBundleImport{}, false, nil + } + if err != nil { + return PeernetBundleImport{}, false, fmt.Errorf("peernet bundle import: get: %w", err) + } + return row, true, nil +} + +func inList(n int) string { return strings.TrimSuffix(strings.Repeat("?,", n), ",") } + +// PendingAmong returns the rows among paths whose full check has not passed +// (state pending), keyed by normalised path. +func (r *PeernetBundleImportRepository) PendingAmong(ctx context.Context, paths []string) (map[string]PeernetBundleImport, error) { + out := map[string]PeernetBundleImport{} + for len(paths) > 0 { + n := min(len(paths), maxBundleImportBatch) + args := make([]any, n) + for i, p := range paths[:n] { + args[i] = normalizeHealthPath(p) + } + paths = paths[n:] + rows, err := r.db.QueryContext(ctx, `SELECT `+bundleImportCols+` FROM peernet_bundle_imports + WHERE state = 'pending' AND file_path IN (`+inList(n)+`)`, args...) + if err != nil { + return nil, fmt.Errorf("peernet bundle import: pending: %w", err) + } + for rows.Next() { + b, err := scanBundleImport(rows) + if err != nil { + _ = rows.Close() + return nil, fmt.Errorf("peernet bundle import: scan: %w", err) + } + out[b.FilePath] = b + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return nil, err + } + _ = rows.Close() + } + return out, nil +} + +// QueueHasPending reports whether any file of the queue item still awaits its +// full check. +func (r *PeernetBundleImportRepository) QueueHasPending(ctx context.Context, queueID int64) (bool, error) { + if queueID <= 0 { + return false, nil + } + var n int + err := r.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM peernet_bundle_imports WHERE queue_id = ? AND state = 'pending'`, queueID).Scan(&n) + if err != nil { + return false, fmt.Errorf("peernet bundle import: queue pending: %w", err) + } + return n > 0, nil +} + +func (r *PeernetBundleImportRepository) transition(ctx context.Context, filePath, to string) (bool, error) { + res, err := r.db.ExecContext(ctx, `UPDATE peernet_bundle_imports SET state = ?, updated_at = ? + WHERE file_path = ? AND state = 'pending'`, to, time.Now().Unix(), normalizeHealthPath(filePath)) + if err != nil { + return false, fmt.Errorf("peernet bundle import: %s: %w", to, err) + } + n, err := res.RowsAffected() + return n > 0, err +} + +// MarkPassed moves a pending file to passed. It reports whether this call did +// it (a file that already passed or failed is left alone). +func (r *PeernetBundleImportRepository) MarkPassed(ctx context.Context, filePath string) (bool, error) { + return r.transition(ctx, filePath, PeernetBundleStatePassed) +} + +// MarkFailed moves a pending file to failed, with the same compare-and-set. +func (r *PeernetBundleImportRepository) MarkFailed(ctx context.Context, filePath string) (bool, error) { + return r.transition(ctx, filePath, PeernetBundleStateFailed) +} + +// ClaimAnnouncement is the single winner's ticket to tell the ARRs about a +// queue item: every file of it passed, the announcement is owed and nobody +// claimed it yet. Concurrent callers cannot both win. It returns one of the +// item's rows, which carries what the notification needs. +func (r *PeernetBundleImportRepository) ClaimAnnouncement(ctx context.Context, queueID int64) (PeernetBundleImport, bool, error) { + if queueID <= 0 { + return PeernetBundleImport{}, false, nil + } + res, err := r.db.ExecContext(ctx, `UPDATE peernet_bundle_imports SET arr_notified = 1, updated_at = ? + WHERE queue_id = ? AND arr_owed = 1 AND arr_notified = 0 + AND NOT EXISTS (SELECT 1 FROM peernet_bundle_imports x WHERE x.queue_id = ? AND (x.state <> 'passed' OR x.committed = 0))`, + time.Now().Unix(), queueID, queueID) + if err != nil { + return PeernetBundleImport{}, false, fmt.Errorf("peernet bundle import: claim: %w", err) + } + if n, _ := res.RowsAffected(); n == 0 { + return PeernetBundleImport{}, false, nil + } + row, err := scanBundleImport(r.db.QueryRowContext(ctx, `SELECT `+bundleImportCols+ + ` FROM peernet_bundle_imports WHERE queue_id = ? ORDER BY file_path LIMIT 1`, queueID)) + if err != nil { + return PeernetBundleImport{}, false, fmt.Errorf("peernet bundle import: claim read: %w", err) + } + return row, true, nil +} + +// ReleaseAnnouncement undoes a claim whose notification could not be sent, so +// a later pass tries again. +func (r *PeernetBundleImportRepository) ReleaseAnnouncement(ctx context.Context, queueID int64) error { + _, err := r.db.ExecContext(ctx, `UPDATE peernet_bundle_imports SET arr_notified = 0 WHERE queue_id = ?`, queueID) + return err +} + +// QueuesAwaitingAnnouncement lists the queue items whose files all passed but +// whose announcement was never claimed (a restart between the two). +func (r *PeernetBundleImportRepository) QueuesAwaitingAnnouncement(ctx context.Context) ([]int64, error) { + rows, err := r.db.QueryContext(ctx, `SELECT DISTINCT queue_id FROM peernet_bundle_imports + WHERE queue_id <> 0 AND arr_owed = 1 AND arr_notified = 0 + AND NOT EXISTS (SELECT 1 FROM peernet_bundle_imports x WHERE x.queue_id = peernet_bundle_imports.queue_id AND (x.state <> 'passed' OR x.committed = 0)) + ORDER BY queue_id LIMIT 1000`) + if err != nil { + return nil, fmt.Errorf("peernet bundle import: awaiting: %w", err) + } + defer rows.Close() + var out []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + out = append(out, id) + } + return out, rows.Err() +} + +// StatesForQueues maps queue ids that have bundle rows to "still pending". +func (r *PeernetBundleImportRepository) StatesForQueues(ctx context.Context, ids []int64) (map[int64]bool, error) { + out := map[int64]bool{} + for len(ids) > 0 { + n := min(len(ids), maxBundleImportBatch) + args := make([]any, n) + for i, id := range ids[:n] { + args[i] = id + } + ids = ids[n:] + rows, err := r.db.QueryContext(ctx, `SELECT queue_id, MAX(CASE WHEN state = 'pending' THEN 1 ELSE 0 END) + FROM peernet_bundle_imports WHERE queue_id IN (`+inList(n)+`) GROUP BY queue_id`, args...) + if err != nil { + return nil, fmt.Errorf("peernet bundle import: queue states: %w", err) + } + for rows.Next() { + var id int64 + var pending int + if err := rows.Scan(&id, &pending); err != nil { + _ = rows.Close() + return nil, err + } + out[id] = pending != 0 + } + err = rows.Err() + _ = rows.Close() + if err != nil { + return nil, err + } + } + return out, nil +} + +// StatesForPaths maps paths that have a bundle row to "still pending". +func (r *PeernetBundleImportRepository) StatesForPaths(ctx context.Context, paths []string) (map[string]bool, error) { + out := map[string]bool{} + for len(paths) > 0 { + n := min(len(paths), maxBundleImportBatch) + args := make([]any, n) + for i, p := range paths[:n] { + args[i] = normalizeHealthPath(p) + } + paths = paths[n:] + rows, err := r.db.QueryContext(ctx, `SELECT file_path, state FROM peernet_bundle_imports + WHERE file_path IN (`+inList(n)+`)`, args...) + if err != nil { + return nil, fmt.Errorf("peernet bundle import: path states: %w", err) + } + for rows.Next() { + var p, state string + if err := rows.Scan(&p, &state); err != nil { + _ = rows.Close() + return nil, err + } + out[p] = state == PeernetBundleStatePending + } + err = rows.Err() + _ = rows.Close() + if err != nil { + return nil, err + } + } + return out, nil +} + +// Delete removes the rows of the given paths. +func (r *PeernetBundleImportRepository) Delete(ctx context.Context, paths []string) error { + for len(paths) > 0 { + n := min(len(paths), maxBundleImportBatch) + args := make([]any, n) + for i, p := range paths[:n] { + args[i] = normalizeHealthPath(p) + } + paths = paths[n:] + if _, err := r.db.ExecContext(ctx, `DELETE FROM peernet_bundle_imports WHERE file_path IN (`+inList(n)+`)`, args...); err != nil { + return fmt.Errorf("peernet bundle import: delete: %w", err) + } + } + return nil +} + +// SetQueueHold sets or clears the queue item's peer_hold mark. A held item +// shows in the SABnzbd-compatible queue as work in progress and never in its +// history; the user's own skip_arr_notification flag is not touched. +func (r *PeernetBundleImportRepository) SetQueueHold(ctx context.Context, queueID int64, held bool) error { + _, err := r.db.ExecContext(ctx, `UPDATE import_queue SET peer_hold = ?, updated_at = ? WHERE id = ?`, + b2i(held), time.Now().UTC(), queueID) + if err != nil { + return fmt.Errorf("peernet bundle import: set hold: %w", err) + } + return nil +} + +// FailHeldQueueItem turns a held queue item into a failed one with a short +// message, so the SABnzbd-compatible history reports it as failed (the ARR then +// blocklists the release and searches again). Its provisional history rows go +// with it, so nothing can later resurface as a completed import. It reports +// whether an item was held and changed. +func (r *PeernetBundleImportRepository) FailHeldQueueItem(ctx context.Context, queueID int64, message string) (bool, error) { + now := time.Now().UTC() + res, err := r.db.ExecContext(ctx, `UPDATE import_queue SET status = 'failed', error_message = ?, peer_hold = 0, + completed_at = ?, updated_at = ? WHERE id = ? AND peer_hold <> 0`, message, now, now, queueID) + if err != nil { + return false, fmt.Errorf("peernet bundle import: fail held item: %w", err) + } + n, err := res.RowsAffected() + if err != nil || n == 0 { + return false, err + } + if _, err := r.db.ExecContext(ctx, `DELETE FROM import_history WHERE nzb_id = ?`, queueID); err != nil { + return true, fmt.Errorf("peernet bundle import: drop history of a failed item: %w", err) + } + return true, nil +} + +// SetQueue links the rows of the given paths to a queue item. +func (r *PeernetBundleImportRepository) SetQueue(ctx context.Context, paths []string, queueID int64) error { + for len(paths) > 0 { + n := min(len(paths), maxBundleImportBatch) + args := []any{queueID} + for _, p := range paths[:n] { + args = append(args, normalizeHealthPath(p)) + } + paths = paths[n:] + if _, err := r.db.ExecContext(ctx, `UPDATE peernet_bundle_imports SET queue_id = ? WHERE file_path IN (`+inList(n)+`)`, args...); err != nil { + return fmt.Errorf("peernet bundle import: set queue: %w", err) + } + } + return nil +} + +// MoveFile makes a row follow its file to a new virtual path. It reports +// whether a row moved. +func (r *PeernetBundleImportRepository) MoveFile(ctx context.Context, from, to string) (bool, error) { + res, err := r.db.ExecContext(ctx, `UPDATE peernet_bundle_imports SET file_path = ?, updated_at = ? WHERE file_path = ?`, + normalizeHealthPath(to), time.Now().Unix(), normalizeHealthPath(from)) + if err != nil { + return false, fmt.Errorf("peernet bundle import: move file: %w", err) + } + n, err := res.RowsAffected() + return n > 0, err +} + +// MoveDir makes the rows below a folder follow it to a new virtual path; the +// recorded resulting folder follows too. +func (r *PeernetBundleImportRepository) MoveDir(ctx context.Context, from, to string) error { + from, to = normalizeHealthPath(from), normalizeHealthPath(to) + if from == "" || to == "" { + return errors.New("peernet bundle import: empty folder") + } + like := escapeLikePrefix(from) + `/%` + _, err := r.db.ExecContext(ctx, `UPDATE peernet_bundle_imports + SET file_path = CAST(? AS TEXT) || substr(file_path, CAST(? AS INTEGER)), updated_at = ? + WHERE file_path LIKE ? ESCAPE '\'`, to, len(from)+1, time.Now().Unix(), like) + if err != nil { + return fmt.Errorf("peernet bundle import: move folder: %w", err) + } + _, err = r.db.ExecContext(ctx, `UPDATE peernet_bundle_imports SET resulting_path = ? WHERE resulting_path = ?`, to, from) + if err != nil { + return fmt.Errorf("peernet bundle import: move folder: %w", err) + } + return nil +} + +// DeleteDir removes the rows below a folder. +func (r *PeernetBundleImportRepository) DeleteDir(ctx context.Context, dir string) error { + dir = normalizeHealthPath(dir) + if dir == "" { + return errors.New("peernet bundle import: empty folder") + } + _, err := r.db.ExecContext(ctx, `DELETE FROM peernet_bundle_imports WHERE file_path LIKE ? ESCAPE '\'`, escapeLikePrefix(dir)+`/%`) + if err != nil { + return fmt.Errorf("peernet bundle import: delete folder: %w", err) + } + return nil +} + +// PendingCounts returns how many files still owe their full check and how many +// of those have waited since before staleBefore (zero time: none counted stale). +func (r *PeernetBundleImportRepository) PendingCounts(ctx context.Context, staleBefore time.Time) (pending, stale int, err error) { + cutoff := int64(0) + if !staleBefore.IsZero() { + cutoff = staleBefore.Unix() + } + err = r.db.QueryRowContext(ctx, `SELECT COUNT(*), COALESCE(SUM(CASE WHEN created_at < ? THEN 1 ELSE 0 END), 0) + FROM peernet_bundle_imports WHERE state = 'pending'`, cutoff).Scan(&pending, &stale) + if err != nil { + return 0, 0, fmt.Errorf("peernet bundle import: pending counts: %w", err) + } + return pending, stale, nil +} + +// StalePending lists pending rows created before staleBefore, oldest first. +func (r *PeernetBundleImportRepository) StalePending(ctx context.Context, staleBefore time.Time, limit int) ([]PeernetBundleImport, error) { + rows, err := r.db.QueryContext(ctx, `SELECT `+bundleImportCols+` FROM peernet_bundle_imports + WHERE state = 'pending' AND created_at < ? ORDER BY created_at LIMIT ?`, staleBefore.Unix(), limit) + if err != nil { + return nil, fmt.Errorf("peernet bundle import: stale pending: %w", err) + } + defer func() { _ = rows.Close() }() + var out []PeernetBundleImport + for rows.Next() { + row, err := scanBundleImport(rows) + if err != nil { + return nil, err + } + out = append(out, row) + } + return out, rows.Err() +} + +// CommitRows marks the rows of the given paths as completely written. +func (r *PeernetBundleImportRepository) CommitRows(ctx context.Context, paths []string) error { + for len(paths) > 0 { + n := min(len(paths), maxBundleImportBatch) + args := []any{time.Now().Unix()} + for _, p := range paths[:n] { + args = append(args, normalizeHealthPath(p)) + } + paths = paths[n:] + if _, err := r.db.ExecContext(ctx, `UPDATE peernet_bundle_imports SET committed = 1, updated_at = ? WHERE file_path IN (`+inList(n)+`)`, args...); err != nil { + return fmt.Errorf("peernet bundle import: commit: %w", err) + } + } + return nil +} + +// ListUncommitted lists the rows of writes that never finished, by path. +func (r *PeernetBundleImportRepository) ListUncommitted(ctx context.Context, limit int) ([]PeernetBundleImport, error) { + return r.listRows(ctx, `WHERE committed = 0 ORDER BY file_path LIMIT ?`, limit) +} + +// ListByQueue lists the rows of one queue item, by path. +func (r *PeernetBundleImportRepository) ListByQueue(ctx context.Context, queueID int64) ([]PeernetBundleImport, error) { + if queueID <= 0 { + return nil, nil + } + return r.listRows(ctx, `WHERE queue_id = ? ORDER BY file_path`, queueID) +} + +// ListAfter lists up to limit rows whose path sorts after the given one, for a +// full walk in pages. +func (r *PeernetBundleImportRepository) ListAfter(ctx context.Context, afterPath string, limit int) ([]PeernetBundleImport, error) { + return r.listRows(ctx, `WHERE file_path > ? ORDER BY file_path LIMIT ?`, afterPath, limit) +} + +func (r *PeernetBundleImportRepository) listRows(ctx context.Context, tail string, args ...any) ([]PeernetBundleImport, error) { + rows, err := r.db.QueryContext(ctx, `SELECT `+bundleImportCols+` FROM peernet_bundle_imports `+tail, args...) + if err != nil { + return nil, fmt.Errorf("peernet bundle import: list: %w", err) + } + defer func() { _ = rows.Close() }() + var out []PeernetBundleImport + for rows.Next() { + row, err := scanBundleImport(rows) + if err != nil { + return nil, err + } + out = append(out, row) + } + return out, rows.Err() +} + +// OrphanedHolds lists the held queue items that no bundle row refers to any +// more (their files were deleted, or the state was lost): nothing can ever +// release such a hold. Only rows carrying the mark are looked at. +func (r *PeernetBundleImportRepository) OrphanedHolds(ctx context.Context) ([]int64, error) { + rows, err := r.db.QueryContext(ctx, `SELECT q.id FROM import_queue q + WHERE q.peer_hold <> 0 + AND NOT EXISTS (SELECT 1 FROM peernet_bundle_imports b WHERE b.queue_id = q.id) + ORDER BY q.id LIMIT 1000`) + if err != nil { + return nil, fmt.Errorf("peernet bundle import: orphaned holds: %w", err) + } + defer func() { _ = rows.Close() }() + var out []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + out = append(out, id) + } + return out, rows.Err() +} diff --git a/internal/database/peernet_bundle_import_repository_test.go b/internal/database/peernet_bundle_import_repository_test.go new file mode 100644 index 000000000..682c9ca61 --- /dev/null +++ b/internal/database/peernet_bundle_import_repository_test.go @@ -0,0 +1,249 @@ +package database + +import ( + "context" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func forEachBundleImportBackend(t *testing.T, fn func(t *testing.T, r *PeernetBundleImportRepository)) { + t.Helper() + db, err := NewDB(Config{DatabasePath: filepath.Join(t.TempDir(), "db")}) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + t.Run("sqlite", func(t *testing.T) { fn(t, NewPeernetBundleImportRepository(db.Connection(), DialectSQLite)) }) + if hasPostgresDSN() { + t.Run("postgres", func(t *testing.T) { + fn(t, NewPeernetBundleImportRepository(openPostgresLatest(t), DialectPostgres)) + }) + } +} + +func hex64(c string) string { return strings.Repeat(c, 64) } + +func bundleRow(path string, queue int64) PeernetBundleImport { + return PeernetBundleImport{ + FilePath: path, FileID: hex64("a"), ReleaseID: hex64("b"), NetworkID: hex64("c"), PublisherID: hex64("d"), + QueueID: queue, ArrOwed: true, Category: "movies", TargetPath: "", ResultingPath: "movies/x", + } +} + +func TestPeernetBundleImport_UpsertGetAndPending(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + require.NoError(t, r.Upsert(ctx, []PeernetBundleImport{bundleRow("/movies/x/a.mkv", 7), bundleRow("movies/x/b.mkv", 7)})) + + got, ok, err := r.Get(ctx, "movies/x/a.mkv") // slash-normalised + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, PeernetBundleStatePending, got.State) + assert.Equal(t, int64(7), got.QueueID) + assert.Equal(t, hex64("d"), got.PublisherID) + assert.True(t, got.ArrOwed) + + _, ok, err = r.Get(ctx, "movies/other.mkv") + require.NoError(t, err) + assert.False(t, ok) + + pend, err := r.PendingAmong(ctx, []string{"movies/x/a.mkv", "movies/other.mkv"}) + require.NoError(t, err) + assert.Len(t, pend, 1) + assert.Contains(t, pend, "movies/x/a.mkv") + + has, err := r.QueueHasPending(ctx, 7) + require.NoError(t, err) + assert.True(t, has) + has, err = r.QueueHasPending(ctx, 8) + require.NoError(t, err) + assert.False(t, has) + }) +} + +func TestPeernetBundleImport_TransitionsAreCompareAndSet(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + require.NoError(t, r.Upsert(ctx, []PeernetBundleImport{bundleRow("m/a.mkv", 1), bundleRow("m/b.mkv", 1)})) + + ok, err := r.MarkPassed(ctx, "m/a.mkv") + require.NoError(t, err) + assert.True(t, ok) + ok, err = r.MarkPassed(ctx, "m/a.mkv") + require.NoError(t, err) + assert.False(t, ok, "second pass is a no-op") + ok, err = r.MarkFailed(ctx, "m/a.mkv") + require.NoError(t, err) + assert.False(t, ok, "a passed file cannot fail") + ok, err = r.MarkFailed(ctx, "m/b.mkv") + require.NoError(t, err) + assert.True(t, ok) + ok, err = r.MarkPassed(ctx, "m/b.mkv") + require.NoError(t, err) + assert.False(t, ok, "a failed file cannot pass") + + pend, err := r.PendingAmong(ctx, []string{"m/a.mkv", "m/b.mkv"}) + require.NoError(t, err) + assert.Empty(t, pend) + }) +} + +func TestPeernetBundleImport_AnnouncementClaimedOnceAfterAllPass(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + require.NoError(t, r.Upsert(ctx, []PeernetBundleImport{bundleRow("m/a.mkv", 5), bundleRow("m/b.mkv", 5)})) + require.NoError(t, r.CommitRows(ctx, []string{"m/a.mkv", "m/b.mkv"})) + + _, claimed, err := r.ClaimAnnouncement(ctx, 5) + require.NoError(t, err) + assert.False(t, claimed, "nothing passed yet") + + _, _ = r.MarkPassed(ctx, "m/a.mkv") + _, claimed, err = r.ClaimAnnouncement(ctx, 5) + require.NoError(t, err) + assert.False(t, claimed, "one file still pending") + + _, _ = r.MarkPassed(ctx, "m/b.mkv") + var wins atomic.Int32 + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + row, ok, err := r.ClaimAnnouncement(ctx, 5) + if err == nil && ok { + wins.Add(1) + assert.Equal(t, "movies", row.Category) + } + }() + } + wg.Wait() + assert.Equal(t, int32(1), wins.Load(), "exactly one claimant wins") + + ids, err := r.QueuesAwaitingAnnouncement(ctx) + require.NoError(t, err) + assert.Empty(t, ids) + }) +} + +func TestPeernetBundleImport_FailedImportIsNeverAnnounced(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + require.NoError(t, r.Upsert(ctx, []PeernetBundleImport{bundleRow("m/a.mkv", 5), bundleRow("m/b.mkv", 5)})) + _, _ = r.MarkPassed(ctx, "m/a.mkv") + _, _ = r.MarkFailed(ctx, "m/b.mkv") + _, claimed, err := r.ClaimAnnouncement(ctx, 5) + require.NoError(t, err) + assert.False(t, claimed) + }) +} + +func TestPeernetBundleImport_NotOwedIsNeverClaimed(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + row := bundleRow("m/a.mkv", 5) + row.ArrOwed = false + require.NoError(t, r.Upsert(ctx, []PeernetBundleImport{row})) + _, _ = r.MarkPassed(ctx, "m/a.mkv") + _, claimed, err := r.ClaimAnnouncement(ctx, 5) + require.NoError(t, err) + assert.False(t, claimed) + }) +} + +func TestPeernetBundleImport_AwaitingAnnouncementSurvivesRestart(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + require.NoError(t, r.Upsert(ctx, []PeernetBundleImport{bundleRow("m/a.mkv", 9)})) + require.NoError(t, r.CommitRows(ctx, []string{"m/a.mkv"})) + _, _ = r.MarkPassed(ctx, "m/a.mkv") + ids, err := r.QueuesAwaitingAnnouncement(ctx) + require.NoError(t, err) + assert.Equal(t, []int64{9}, ids) + }) +} + +func TestPeernetBundleImport_QueueAndPathSummaries(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + require.NoError(t, r.Upsert(ctx, []PeernetBundleImport{bundleRow("m/a.mkv", 3), bundleRow("n/c.mkv", 4)})) + _, _ = r.MarkPassed(ctx, "n/c.mkv") + + byQueue, err := r.StatesForQueues(ctx, []int64{3, 4, 99}) + require.NoError(t, err) + assert.Equal(t, map[int64]bool{3: true, 4: false}, byQueue, "value is pending") + + byPath, err := r.StatesForPaths(ctx, []string{"m/a.mkv", "n/c.mkv", "zzz"}) + require.NoError(t, err) + assert.Equal(t, map[string]bool{"m/a.mkv": true, "n/c.mkv": false}, byPath) + + empty, err := r.StatesForPaths(ctx, nil) + require.NoError(t, err) + assert.Empty(t, empty) + }) +} + +func TestPeernetBundleImport_DeleteAndQueueSkipFlag(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + require.NoError(t, r.Upsert(ctx, []PeernetBundleImport{bundleRow("m/a.mkv", 3)})) + require.NoError(t, r.Delete(ctx, []string{"/m/a.mkv"})) + _, ok, err := r.Get(ctx, "m/a.mkv") + require.NoError(t, err) + assert.False(t, ok) + }) +} + +func TestPeernetBundleImport_SetQueueLinksRows(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + require.NoError(t, r.Upsert(ctx, []PeernetBundleImport{bundleRow("m/a.mkv", 0), bundleRow("m/b.mkv", 0)})) + require.NoError(t, r.SetQueue(ctx, []string{"/m/a.mkv", "m/b.mkv"}, 42)) + has, err := r.QueueHasPending(ctx, 42) + require.NoError(t, err) + assert.True(t, has) + }) +} + +func TestPeernetBundleImport_MoveDeleteAndPendingCounts(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + a, b, other := bundleRow("movies/x_y/a.mkv", 7), bundleRow("movies/x_y/b.mkv", 7), bundleRow("movies/xzy/c.mkv", 8) + a.ResultingPath, b.ResultingPath, other.ResultingPath = "movies/x_y", "movies/x_y", "movies/xzy" + require.NoError(t, r.Upsert(ctx, []PeernetBundleImport{a, b, other})) + + moved, err := r.MoveFile(ctx, "movies/x_y/a.mkv", "movies/x_y/renamed.mkv") + require.NoError(t, err) + assert.True(t, moved) + _, ok, _ := r.Get(ctx, "movies/x_y/a.mkv") + assert.False(t, ok) + + require.NoError(t, r.MoveDir(ctx, "movies/x_y", "movies/new")) + got, ok, err := r.Get(ctx, "movies/new/b.mkv") + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, "movies/new", got.ResultingPath) + _, ok, _ = r.Get(ctx, "movies/xzy/c.mkv") + assert.True(t, ok, "'_' is a literal, not a wildcard: the sibling folder is untouched") + + pending, stale, err := r.PendingCounts(ctx, time.Now().Add(time.Hour)) + require.NoError(t, err) + assert.Equal(t, 3, pending) + assert.Equal(t, 3, stale) + pending, stale, err = r.PendingCounts(ctx, time.Now().Add(-time.Hour)) + require.NoError(t, err) + assert.Equal(t, 3, pending) + assert.Zero(t, stale) + + require.NoError(t, r.DeleteDir(ctx, "movies/new")) + _, ok, _ = r.Get(ctx, "movies/new/b.mkv") + assert.False(t, ok) + _, ok, _ = r.Get(ctx, "movies/xzy/c.mkv") + assert.True(t, ok) + }) +} diff --git a/internal/database/peernet_bundle_queries.go b/internal/database/peernet_bundle_queries.go new file mode 100644 index 000000000..5b575e43e --- /dev/null +++ b/internal/database/peernet_bundle_queries.go @@ -0,0 +1,171 @@ +package database + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" +) + +// Bundle query bounds. +const ( + // PeernetBundleOffersMaxLimit caps BundleOffers results. + PeernetBundleOffersMaxLimit = 100 + // PeernetBundleMismatchMaxLimit caps ListBundleLevelMismatch results. + PeernetBundleMismatchMaxLimit = 200 +) + +// PeernetBundleOffer is one stored release record that announces a bundle. +// The Bundle ID comes from the signed record. +type PeernetBundleOffer struct { + NetworkID string + PublisherNodeID string + BundleID string + // ReleaseScheme is PeernetSchemeNZB or PeernetSchemeFiles: how the record's + // release ID was derived, and so how the bundle is verified. + ReleaseScheme int + CreatedAt time.Time + Source int +} + +// BundleOffers returns the non-expired release records of releaseID that +// announce a bundle, across the given networks (at most +// PeernetMaxNetworksPerQuery), newest first and at most +// min(limit, PeernetBundleOffersMaxLimit). It is served by the +// (network, release, kind) index. +func (r *PeernetRecordRepository) BundleOffers(ctx context.Context, networkIDs []string, releaseID string, limit int) ([]PeernetBundleOffer, error) { + switch { + case !isLowerHexID(releaseID): + return nil, errors.New("peernet records: invalid release id") + case len(networkIDs) == 0: + return nil, errors.New("peernet records: no networks") + case len(networkIDs) > PeernetMaxNetworksPerQuery: + return nil, ErrPeernetTooManyIDs + case limit < 1: + return nil, errors.New("peernet records: limit must be positive") + } + limit = min(limit, PeernetBundleOffersMaxLimit) + args := make([]any, 0, len(networkIDs)+4) + for _, n := range networkIDs { + if !isLowerHexID(n) { + return nil, errors.New("peernet records: invalid network id") + } + args = append(args, n) + } + args = append(args, releaseID, PeernetRecordKindRelease, r.now().Unix(), limit) + rows, err := r.db.QueryContext(ctx, + `SELECT network_id, publisher_node_id, bundle_id, release_scheme, created_at, source + FROM peernet_records + WHERE network_id IN (?`+strings.Repeat(",?", len(networkIDs)-1)+`) AND release_id = ? AND kind = ? + AND bundle_id <> '' AND expires_at > ? + ORDER BY created_at DESC, seq DESC, network_id LIMIT ?`, args...) + if err != nil { + return nil, fmt.Errorf("query peernet bundle offers: %w", err) + } + defer rows.Close() + var out []PeernetBundleOffer + for rows.Next() { + var o PeernetBundleOffer + var created int64 + if err := rows.Scan(&o.NetworkID, &o.PublisherNodeID, &o.BundleID, &o.ReleaseScheme, &created, &o.Source); err != nil { + return nil, fmt.Errorf("scan peernet bundle offer: %w", err) + } + o.CreatedAt = timeOrZero(created) + out = append(out, o) + } + return out, rows.Err() +} + +// GetReleaseByBundle returns the release-record state of the release this node +// announced in network with the given Bundle ID. It is the only way a bundle +// request is matched: an ID this node did not announce there is not found. +func (r *PeernetPublishStateRepository) GetReleaseByBundle(ctx context.Context, network, bundleID string) (PeernetPublishState, bool, error) { + if !isLowerHexID(network) || !isLowerHexID(bundleID) { + return PeernetPublishState{}, false, errors.New("peernet publish state: invalid network or bundle id") + } + s := PeernetPublishState{NetworkID: network, Kind: PeernetRecordKindRelease, BundleID: bundleID} + var published int64 + err := r.db.QueryRowContext(ctx, + `SELECT release_id, published_at, origin_category, origin_indexer, bundle_level FROM peernet_publish_state + WHERE network_id = ? AND bundle_id = ? AND kind = ? ORDER BY published_at DESC LIMIT 1`, + network, bundleID, PeernetRecordKindRelease). + Scan(&s.ReleaseID, &published, &s.OriginCategory, &s.OriginIndexer, &s.BundleLevel) + if errors.Is(err, sql.ErrNoRows) { + return PeernetPublishState{}, false, nil + } + if err != nil { + return PeernetPublishState{}, false, fmt.Errorf("get peernet publish state by bundle: %w", err) + } + s.PublishedAt = timeOrZero(published) + return s, true, nil +} + +// ListBundleLevelMismatch returns up to limit release-record states of network +// whose publication does not match the network's current level: with +// wantBundle set (level 3) the records published without a bundle decision, +// with it clear (level below 3) the records published at level 3. These are the +// records to announce again. Oldest first. +func (r *PeernetPublishStateRepository) ListBundleLevelMismatch(ctx context.Context, network string, wantBundle bool, limit int) ([]PeernetPublishState, error) { + if !isLowerHexID(network) { + return nil, errors.New("peernet publish state: invalid network id") + } + if limit < 1 { + return nil, errors.New("peernet publish state: limit must be positive") + } + limit = min(limit, PeernetBundleMismatchMaxLimit) + cond := `bundle_level >= ?` + if wantBundle { + cond = `bundle_level < ?` + } + rows, err := r.db.QueryContext(ctx, + `SELECT release_id, published_at, bundle_id, bundle_level FROM peernet_publish_state + WHERE network_id = ? AND kind = ? AND `+cond+` ORDER BY published_at, release_id LIMIT ?`, + network, PeernetRecordKindRelease, PeernetBundleLevel, limit) + if err != nil { + return nil, fmt.Errorf("list peernet bundle level mismatch: %w", err) + } + defer rows.Close() + var out []PeernetPublishState + for rows.Next() { + s := PeernetPublishState{NetworkID: network, Kind: PeernetRecordKindRelease} + var published int64 + if err := rows.Scan(&s.ReleaseID, &published, &s.BundleID, &s.BundleLevel); err != nil { + return nil, fmt.Errorf("scan peernet bundle level mismatch: %w", err) + } + s.PublishedAt = timeOrZero(published) + out = append(out, s) + } + return out, rows.Err() +} + +// PeernetBundleLevel is the exposure level at which bundles are announced. +const PeernetBundleLevel = 3 + +// MarkBundleStale clears the bundle decision of a release record's state, so +// the next level-3 sweep announces it again. The announced Bundle ID stays +// until the new record replaces it. +func (r *PeernetPublishStateRepository) MarkBundleStale(ctx context.Context, network, release string) error { + if !isLowerHexID(network) || !isLowerHexID(release) { + return errors.New("peernet publish state: invalid network or release id") + } + if _, err := r.db.ExecContext(ctx, + `UPDATE peernet_publish_state SET bundle_level = 0 WHERE network_id = ? AND release_id = ? AND kind = ?`, + network, release, PeernetRecordKindRelease); err != nil { + return fmt.Errorf("mark peernet bundle stale: %w", err) + } + return nil +} + +// CountAnnouncedBundles returns how many release records this node announced +// with a Bundle ID, over every network. +func (r *PeernetPublishStateRepository) CountAnnouncedBundles(ctx context.Context) (int, error) { + var n int + if err := r.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM peernet_publish_state WHERE kind = ? AND bundle_id <> ''`, + PeernetRecordKindRelease).Scan(&n); err != nil { + return 0, fmt.Errorf("count peernet announced bundles: %w", err) + } + return n, nil +} diff --git a/internal/database/peernet_bundle_test.go b/internal/database/peernet_bundle_test.go new file mode 100644 index 000000000..dcdfd6cb0 --- /dev/null +++ b/internal/database/peernet_bundle_test.go @@ -0,0 +1,176 @@ +package database + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func bundleHex(i int) string { return relHex(i + 7_000_000) } + +func TestPeernetPublishState_BundleRoundTripAndLookup(t *testing.T) { + forEachStateBackend(t, func(t *testing.T, b *stateBackend) { + ctx := context.Background() + in := st(hexID('a'), PeernetRecordKindRelease, hexID('1'), "", 1000) + in.BundleID, in.BundleLevel = bundleHex(1), 3 + in.OriginCategory, in.OriginIndexer = "Movies", "IdxOne" + require.NoError(t, b.state.Put(ctx, in)) + + got, ok, err := b.state.Get(ctx, hexID('a'), PeernetRecordKindRelease, hexID('1'), "") + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, bundleHex(1), got.BundleID) + assert.Equal(t, 3, got.BundleLevel) + + byBundle, ok, err := b.state.GetReleaseByBundle(ctx, hexID('a'), bundleHex(1)) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, hexID('1'), byBundle.ReleaseID) + assert.Equal(t, "movies", byBundle.OriginCategory) + + _, ok, err = b.state.GetReleaseByBundle(ctx, hexID('b'), bundleHex(1)) + require.NoError(t, err) + assert.False(t, ok, "announcements are per network") + _, ok, err = b.state.GetReleaseByBundle(ctx, hexID('a'), bundleHex(2)) + require.NoError(t, err) + assert.False(t, ok, "an unknown bundle id is not announced") + _, _, err = b.state.GetReleaseByBundle(ctx, hexID('a'), "nothex") + require.Error(t, err) + _, ok, err = b.state.GetReleaseByBundle(ctx, hexID('a'), "") + assert.False(t, ok) + _ = err + }) +} + +func TestPeernetPublishState_BundleLevelMismatchAndStale(t *testing.T) { + forEachStateBackend(t, func(t *testing.T, b *stateBackend) { + ctx := context.Background() + mk := func(rel string, bundle string, level int) { + s := st(hexID('a'), PeernetRecordKindRelease, rel, "", 10) + s.BundleID, s.BundleLevel = bundle, level + require.NoError(t, b.state.Put(ctx, s)) + } + mk(hexID('1'), bundleHex(1), 3) // announced with a bundle + mk(hexID('2'), "", 3) // decided at level 3: not shareable + mk(hexID('3'), "", 2) // published at level 2 + att := st(hexID('a'), PeernetRecordKindAttestation, hexID('1'), hexID('f'), 10) + require.NoError(t, b.state.Put(ctx, att)) + + // Network now at level 3: only the level-2 record needs re-announcing. + got, err := b.state.ListBundleLevelMismatch(ctx, hexID('a'), true, 10) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, hexID('3'), got[0].ReleaseID) + + // Network dropped to level 2: both level-3 records must be superseded. + got, err = b.state.ListBundleLevelMismatch(ctx, hexID('a'), false, 10) + require.NoError(t, err) + require.Len(t, got, 2) + + got, err = b.state.ListBundleLevelMismatch(ctx, hexID('a'), false, 1) + require.NoError(t, err) + assert.Len(t, got, 1, "bounded") + + // A stale bundle is marked so the next level-3 sweep re-announces it. + require.NoError(t, b.state.MarkBundleStale(ctx, hexID('a'), hexID('1'))) + got, err = b.state.ListBundleLevelMismatch(ctx, hexID('a'), true, 10) + require.NoError(t, err) + assert.Len(t, got, 2) + n, err := b.state.CountAnnouncedBundles(ctx) + require.NoError(t, err) + assert.Equal(t, 1, n, "stale marking leaves the announcement until it is replaced") + }) +} + +func TestPeernetRecords_BundleOffers(t *testing.T) { + forEachRecBackend(t, func(t *testing.T, b *recBackend) { + ctx := context.Background() + rel := relHex(1) + insertOK(t, b, mkRec(func(r *PeernetRecord) { r.ReleaseID, r.PublisherNodeID, r.BundleID = rel, nodeHex(1), bundleHex(1) })) + insertOK(t, b, mkRec(func(r *PeernetRecord) { + r.ReleaseID, r.PublisherNodeID, r.BundleID, r.ReleaseScheme = rel, nodeHex(2), bundleHex(2), PeernetSchemeFiles + r.CreatedAt = recT0.Add(time.Minute) + })) + insertOK(t, b, mkRec(func(r *PeernetRecord) { r.ReleaseID, r.PublisherNodeID = rel, nodeHex(3) })) // no bundle + insertOK(t, b, mkRec(func(r *PeernetRecord) { + r.ReleaseID, r.PublisherNodeID, r.BundleID = relHex(9), nodeHex(4), bundleHex(9) + })) // other release + insertOK(t, b, mkRec(func(r *PeernetRecord) { + r.NetworkID, r.ReleaseID, r.PublisherNodeID, r.BundleID = hexID('b'), rel, nodeHex(5), bundleHex(5) + })) + insertOK(t, b, mkRec(func(r *PeernetRecord) { + r.ReleaseID, r.PublisherNodeID, r.BundleID, r.Source, r.RelayerNodeID = rel, nodeHex(6), bundleHex(6), PeernetRecordSourceLocal, "" + })) + + got, err := b.recs.BundleOffers(ctx, []string{hexID('a')}, rel, 10) + require.NoError(t, err) + require.Len(t, got, 3, "network a, this release, with a bundle id; the local record is listed too") + assert.Equal(t, nodeHex(2), got[0].PublisherNodeID, "newest creation time first") + assert.Equal(t, nodeHex(6), got[1].PublisherNodeID, "then newest seq among equal times") + byPub := map[string]PeernetBundleOffer{} + for _, o := range got { + byPub[o.PublisherNodeID] = o + } + assert.Equal(t, bundleHex(2), byPub[nodeHex(2)].BundleID) + assert.Equal(t, PeernetSchemeFiles, byPub[nodeHex(2)].ReleaseScheme) + assert.Equal(t, PeernetSchemeNZB, byPub[nodeHex(1)].ReleaseScheme) + assert.Equal(t, PeernetRecordSourceLocal, byPub[nodeHex(6)].Source) + + got, err = b.recs.BundleOffers(ctx, []string{hexID('a'), hexID('b')}, rel, 10) + require.NoError(t, err) + assert.Len(t, got, 4) + got, err = b.recs.BundleOffers(ctx, []string{hexID('a'), hexID('b')}, rel, 2) + require.NoError(t, err) + assert.Len(t, got, 2, "bounded") + + // Expired records are not offers. + b.setNow(recT0.Add(48 * time.Hour)) + got, err = b.recs.BundleOffers(ctx, []string{hexID('a')}, rel, 10) + require.NoError(t, err) + assert.Empty(t, got) + + _, err = b.recs.BundleOffers(ctx, nil, rel, 10) + require.Error(t, err) + _, err = b.recs.BundleOffers(ctx, []string{hexID('a')}, "bad", 10) + require.Error(t, err) + }) +} + +func TestPeernetRecords_BundleIDValidation(t *testing.T) { + forEachRecBackend(t, func(t *testing.T, b *recBackend) { + _, err := b.recs.Insert(context.Background(), mkRec(func(r *PeernetRecord) { r.BundleID = "short" }), PeernetRecordLimits{}) + require.ErrorIs(t, err, ErrPeernetRecordInvalid) + _, err = b.recs.Insert(context.Background(), mkAtt(true, func(r *PeernetRecord) { r.BundleID = bundleHex(1) }), PeernetRecordLimits{}) + require.ErrorIs(t, err, ErrPeernetRecordInvalid, "an attestation never carries a bundle id") + }) +} + +// AnyBundleOffer answers the cheap "does this node hold any bundle offer" +// question, and the generation counter moves exactly when a record announcing +// a bundle is stored (what invalidates the cached answer). +func TestPeernetRecords_AnyBundleOfferAndGeneration(t *testing.T) { + forEachRecBackend(t, func(t *testing.T, b *recBackend) { + ctx := context.Background() + any, err := b.recs.AnyBundleOffer(ctx) + require.NoError(t, err) + assert.False(t, any) + + gen := BundleOfferGeneration() + insertOK(t, b, mkRec(func(r *PeernetRecord) { r.ReleaseID, r.PublisherNodeID = relHex(1), nodeHex(1) })) // no bundle + assert.Equal(t, gen, BundleOfferGeneration(), "a record without a bundle changes nothing") + any, err = b.recs.AnyBundleOffer(ctx) + require.NoError(t, err) + assert.False(t, any) + + insertOK(t, b, mkRec(func(r *PeernetRecord) { + r.ReleaseID, r.PublisherNodeID, r.BundleID = relHex(2), nodeHex(2), bundleHex(2) + })) + assert.Greater(t, BundleOfferGeneration(), gen, "a bundle-announcing record moves the generation") + any, err = b.recs.AnyBundleOffer(ctx) + require.NoError(t, err) + assert.True(t, any) + }) +} diff --git a/internal/database/peernet_content_derive.go b/internal/database/peernet_content_derive.go new file mode 100644 index 000000000..dae0e17c3 --- /dev/null +++ b/internal/database/peernet_content_derive.go @@ -0,0 +1,364 @@ +package database + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + "unicode/utf8" +) + +// peernetVideoExts are the file extensions the "healthy on the network" label +// treats as a release's main files. +var peernetVideoExts = []string{ + "mkv", "mp4", "m4v", "avi", "mov", "wmv", "mpg", "mpeg", "ts", "m2ts", "webm", "flv", "vob", +} + +// GroupSummaries returns up to limit import groups after the given key, in key +// order, each with the number of its files that are not nzb-scheme. It is a +// keyset scan over the group index, so memory is bounded by limit and the +// caller can stop and resume at any key. +func (r *PeernetContentRepository) GroupSummaries(ctx context.Context, after string, limit int) ([]PeernetGroupSummary, error) { + if limit < 1 { + return nil, errors.New("peernet content: group limit must be positive") + } + rows, err := r.db.QueryContext(ctx, + `SELECT group_key, COUNT(*) FROM peernet_content_index + WHERE group_key > ? AND group_key <> '' AND release_scheme <> 1 + GROUP BY group_key ORDER BY group_key LIMIT ?`, after, limit) + if err != nil { + return nil, fmt.Errorf("list peernet groups: %w", err) + } + defer rows.Close() + var out []PeernetGroupSummary + for rows.Next() { + var g PeernetGroupSummary + if err := rows.Scan(&g.Key, &g.Files); err != nil { + return nil, fmt.Errorf("scan peernet group: %w", err) + } + out = append(out, g) + } + return out, rows.Err() +} + +// GroupMembers returns the non-nzb-scheme rows of the given groups, ordered by +// group key and then path. The caller bounds the total by the group sizes it +// asked for from GroupSummaries. +func (r *PeernetContentRepository) GroupMembers(ctx context.Context, keys []string) ([]PeernetContentEntry, error) { + sorted := make([]string, 0, len(keys)) + for _, k := range keys { + if !isHexID(k) { + return nil, errors.New("peernet content: invalid group key") + } + sorted = append(sorted, strings.ToLower(k)) + } + sort.Strings(sorted) + var out []PeernetContentEntry + for start := 0; start < len(sorted); start += peernetMissingChunk { + chunk := sorted[start:min(start+peernetMissingChunk, len(sorted))] + args := make([]any, len(chunk)) + for i, k := range chunk { + args[i] = k + } + rows, err := r.db.QueryContext(ctx, + `SELECT `+peernetContentCols+` FROM peernet_content_index + WHERE release_scheme <> 1 AND group_key IN (?`+strings.Repeat(",?", len(chunk)-1)+`) + ORDER BY group_key, file_path`, args...) + if err != nil { + return nil, fmt.Errorf("list peernet group members: %w", err) + } + for rows.Next() { + e, err := scanContent(rows) + if err != nil { + rows.Close() + return nil, fmt.Errorf("scan peernet group member: %w", err) + } + out = append(out, e) + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + } + return out, nil +} + +// ApplyDerived writes derived release IDs in one short transaction and returns +// how many rows changed. A row is only touched while it is still the row the +// caller read (same file ID and group key) and is not nzb-scheme, so a +// concurrent re-index or a store-based ID always wins. An empty ReleaseID +// clears the row's derived ID. +func (r *PeernetContentRepository) ApplyDerived(ctx context.Context, updates []PeernetDerivedUpdate) (int, error) { + if len(updates) == 0 { + return 0, nil + } + for _, u := range updates { + if !isHexID(u.FileID) || !isHexID(u.GroupKey) || (u.ReleaseID != "" && !isHexID(u.ReleaseID)) { + return 0, errors.New("peernet content: invalid derived update") + } + } + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return 0, fmt.Errorf("apply derived release ids: begin: %w", err) + } + defer tx.Rollback() //nolint:errcheck // no-op after commit + stmt, err := tx.PrepareContext(ctx, + `UPDATE peernet_content_index SET release_id = ?, release_scheme = ? + WHERE file_path = ? AND file_id = ? AND group_key = ? AND release_scheme <> 1 + AND (release_id <> ? OR release_scheme <> ?)`) + if err != nil { + return 0, fmt.Errorf("apply derived release ids: prepare: %w", err) + } + defer stmt.Close() + changed := 0 + for _, u := range updates { + rel, scheme := strings.ToLower(u.ReleaseID), PeernetSchemeNone + if rel != "" { + scheme = PeernetSchemeFiles + } + res, err := stmt.ExecContext(ctx, rel, scheme, normalizePeernetPath(u.FilePath), + strings.ToLower(u.FileID), strings.ToLower(u.GroupKey), rel, scheme) + if err != nil { + return 0, fmt.Errorf("apply derived release id %q: %w", u.FilePath, err) + } + n, err := rowsAffected(res, "apply derived release id") + if err != nil { + return 0, err + } + changed += int(n) + } + if err := tx.Commit(); err != nil { + return 0, fmt.Errorf("apply derived release ids: commit: %w", err) + } + return changed, nil +} + +// StillIndexed reports which of the given release IDs are carried by at least +// one indexed file. +func (r *PeernetContentRepository) StillIndexed(ctx context.Context, releaseIDs []string) (map[string]bool, error) { + out := make(map[string]bool, len(releaseIDs)) + ids := make([]string, 0, len(releaseIDs)) + for _, id := range releaseIDs { + if !isHexID(id) { + return nil, errors.New("peernet content: invalid release id") + } + ids = append(ids, strings.ToLower(id)) + } + for start := 0; start < len(ids); start += peernetMissingChunk { + chunk := ids[start:min(start+peernetMissingChunk, len(ids))] + args := make([]any, len(chunk)) + for i, id := range chunk { + args[i] = id + } + rows, err := r.db.QueryContext(ctx, + `SELECT DISTINCT release_id FROM peernet_content_index WHERE release_id IN (?`+strings.Repeat(",?", len(chunk)-1)+`)`, args...) + if err != nil { + return nil, fmt.Errorf("query indexed peernet releases: %w", err) + } + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + rows.Close() + return nil, fmt.Errorf("scan indexed release: %w", err) + } + out[id] = true + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + } + return out, nil +} + +// RefsByPrefix returns the group and release identity of dir and everything +// beneath it, at most limit rows. It refuses an empty directory. +func (r *PeernetContentRepository) RefsByPrefix(ctx context.Context, dir string, limit int) ([]PeernetContentRef, error) { + d := normalizePeernetPath(dir) + if d == "" { + return nil, errors.New("peernet content: empty directory prefix") + } + if limit < 1 { + return nil, errors.New("peernet content: limit must be positive") + } + pre := d + "/" + rows, err := r.db.QueryContext(ctx, + `SELECT group_key, release_id, release_scheme FROM peernet_content_index + WHERE file_path = ? OR substr(file_path, 1, ?) = ? LIMIT ?`, + d, utf8.RuneCountInString(pre), pre, limit) + if err != nil { + return nil, fmt.Errorf("list peernet content under %q: %w", d, err) + } + defer rows.Close() + var out []PeernetContentRef + for rows.Next() { + var ref PeernetContentRef + if err := rows.Scan(&ref.GroupKey, &ref.ReleaseID, &ref.ReleaseScheme); err != nil { + return nil, fmt.Errorf("scan peernet content ref: %w", err) + } + out = append(out, ref) + } + return out, rows.Err() +} + +// RefByPath returns the group and release identity of one path. +func (r *PeernetContentRepository) RefByPath(ctx context.Context, filePath string) (PeernetContentRef, bool, error) { + e, ok, err := r.GetByPath(ctx, filePath) + if err != nil || !ok { + return PeernetContentRef{}, false, err + } + return PeernetContentRef{GroupKey: e.GroupKey, ReleaseID: e.ReleaseID, ReleaseScheme: e.ReleaseScheme}, true, nil +} + +// VideoFilesOfReleases returns, per release, the File IDs of its video files +// in path order, at most perRelease+1 each (the extra one tells the caller the +// release has more than it can label). It is one query for up to +// PeernetMaxPathsPerQuery releases. Releases without a video file are absent. +func (r *PeernetContentRepository) VideoFilesOfReleases(ctx context.Context, releaseIDs []string, perRelease int) (map[string][]string, error) { + if perRelease < 1 { + return nil, errors.New("peernet content: per-release limit must be positive") + } + seen := make(map[string]struct{}, len(releaseIDs)) + ids := make([]string, 0, len(releaseIDs)) + for _, id := range releaseIDs { + if _, dup := seen[id]; dup { + continue + } + seen[id] = struct{}{} + if !isHexID(id) { + return nil, errors.New("peernet content: invalid release id") + } + ids = append(ids, strings.ToLower(id)) + } + if len(ids) > PeernetMaxPathsPerQuery { + return nil, ErrPeernetTooManyIDs + } + out := make(map[string][]string, len(ids)) + if len(ids) == 0 { + return out, nil + } + args := make([]any, 0, len(ids)+1) + for _, id := range ids { + args = append(args, id) + } + args = append(args, perRelease+1) + conds := make([]string, len(peernetVideoExts)) + for i, e := range peernetVideoExts { + conds[i] = `lower(file_path) LIKE '%.` + e + `'` // e is a compile-time literal + } + rows, err := r.db.QueryContext(ctx, + `SELECT release_id, file_id FROM ( + SELECT release_id, file_id, ROW_NUMBER() OVER (PARTITION BY release_id ORDER BY file_path) AS rn + FROM peernet_content_index + WHERE release_id IN (?`+strings.Repeat(",?", len(ids)-1)+`) AND (`+strings.Join(conds, " OR ")+`) + ) ranked WHERE rn <= ? ORDER BY release_id, rn`, args...) + if err != nil { + return nil, fmt.Errorf("query peernet release video files: %w", err) + } + defer rows.Close() + for rows.Next() { + var rel, file string + if err := rows.Scan(&rel, &file); err != nil { + return nil, fmt.Errorf("scan peernet release video file: %w", err) + } + out[rel] = append(out[rel], file) + } + return out, rows.Err() +} + +// GroupCounts returns, for the given groups, how many files are not +// nzb-scheme. Empty groups are absent. It lets a caller refuse a huge group +// before reading its members. +func (r *PeernetContentRepository) GroupCounts(ctx context.Context, keys []string) (map[string]int, error) { + out := make(map[string]int, len(keys)) + norm := make([]string, 0, len(keys)) + for _, k := range keys { + if !isHexID(k) { + return nil, errors.New("peernet content: invalid group key") + } + norm = append(norm, strings.ToLower(k)) + } + for start := 0; start < len(norm); start += peernetMissingChunk { + chunk := norm[start:min(start+peernetMissingChunk, len(norm))] + args := make([]any, len(chunk)) + for i, k := range chunk { + args[i] = k + } + rows, err := r.db.QueryContext(ctx, + `SELECT group_key, COUNT(*) FROM peernet_content_index + WHERE release_scheme <> 1 AND group_key IN (?`+strings.Repeat(",?", len(chunk)-1)+`) + GROUP BY group_key`, args...) + if err != nil { + return nil, fmt.Errorf("count peernet groups: %w", err) + } + for rows.Next() { + var k string + var n int + if err := rows.Scan(&k, &n); err != nil { + rows.Close() + return nil, fmt.Errorf("scan peernet group count: %w", err) + } + out[k] = n + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + } + return out, nil +} + +// peernetClearBatch is the most rows ClearDerivedGroup updates per transaction. +const peernetClearBatch = 1000 + +// ClearDerivedGroup removes the derived release ID from every row of one +// group, in short transactions of at most batch rows (default 1000), and +// returns the distinct IDs it removed. nzb-scheme rows are never touched. +func (r *PeernetContentRepository) ClearDerivedGroup(ctx context.Context, key string, batch int) ([]string, error) { + if !isHexID(key) { + return nil, errors.New("peernet content: invalid group key") + } + if batch < 1 { + batch = peernetClearBatch + } + key = strings.ToLower(key) + rows, err := r.db.QueryContext(ctx, + `SELECT DISTINCT release_id FROM peernet_content_index WHERE group_key = ? AND release_scheme = 2`, key) + if err != nil { + return nil, fmt.Errorf("list derived release ids: %w", err) + } + var old []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + rows.Close() + return nil, fmt.Errorf("scan derived release id: %w", err) + } + old = append(old, id) + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + for { + res, err := r.db.ExecContext(ctx, + `UPDATE peernet_content_index SET release_id = '', release_scheme = 0 + WHERE file_path IN (SELECT file_path FROM peernet_content_index + WHERE group_key = ? AND release_scheme = 2 LIMIT ?)`, key, batch) + if err != nil { + return old, fmt.Errorf("clear derived release ids: %w", err) + } + n, err := rowsAffected(res, "clear derived release ids") + if err != nil { + return old, err + } + if int(n) < batch { + return old, nil + } + } +} diff --git a/internal/database/peernet_content_derive_test.go b/internal/database/peernet_content_derive_test.go new file mode 100644 index 000000000..e616c41f6 --- /dev/null +++ b/internal/database/peernet_content_derive_test.go @@ -0,0 +1,214 @@ +package database + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func gk(c byte) string { return hexID(c) } + +func TestPeernetContent_SchemeAndGroupRoundTrip(t *testing.T) { + r := newPeernetTestRepo(t) + ctx := context.Background() + require.NoError(t, r.UpsertBatch(ctx, []PeernetContentEntry{ + {FilePath: "c/a.mkv", FileID: hexID('a'), ReleaseID: hexID('1'), ReleaseScheme: PeernetSchemeNZB, GroupKey: gk('d')}, + {FilePath: "c/b.mkv", FileID: hexID('b'), ReleaseID: hexID('2'), ReleaseScheme: PeernetSchemeFiles, GroupKey: gk('e')}, + {FilePath: "c/c.mkv", FileID: hexID('c')}, + })) + a, ok, err := r.GetByPath(ctx, "c/a.mkv") + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, PeernetSchemeNZB, a.ReleaseScheme) + assert.Equal(t, gk('d'), a.GroupKey) + b, _, _ := r.GetByPath(ctx, "c/b.mkv") + assert.Equal(t, PeernetSchemeFiles, b.ReleaseScheme) + c, _, _ := r.GetByPath(ctx, "c/c.mkv") + assert.Equal(t, 0, c.ReleaseScheme) + assert.Empty(t, c.GroupKey) + + page, err := r.ListPage(ctx, "", 10) + require.NoError(t, err) + require.Len(t, page, 3) + assert.Equal(t, PeernetSchemeFiles, page[1].ReleaseScheme) + byRel, err := r.FindByReleaseID(ctx, hexID('2')) + require.NoError(t, err) + require.Len(t, byRel, 1) + assert.Equal(t, PeernetSchemeFiles, byRel[0].ReleaseScheme) +} + +func TestPeernetContent_RejectsInconsistentScheme(t *testing.T) { + r := newPeernetTestRepo(t) + ctx := context.Background() + bad := []PeernetContentEntry{ + {FilePath: "x", FileID: hexID('a'), ReleaseScheme: PeernetSchemeFiles}, // scheme without id + {FilePath: "x", FileID: hexID('a'), ReleaseID: hexID('1'), ReleaseScheme: 3}, // unknown scheme + {FilePath: "x", FileID: hexID('a'), GroupKey: "not-hex"}, // bad group key + {FilePath: "x", FileID: hexID('a'), ReleaseID: hexID('1'), ReleaseScheme: -1}, // negative + } + for i, e := range bad { + assert.Error(t, r.Upsert(ctx, e), "case %d", i) + } + // A release ID with no scheme is the original (nzb) kind. + require.NoError(t, r.Upsert(ctx, PeernetContentEntry{FilePath: "x", FileID: hexID('a'), ReleaseID: hexID('1')})) + got, _, _ := r.GetByPath(ctx, "x") + assert.Equal(t, PeernetSchemeNZB, got.ReleaseScheme) +} + +func TestPeernetContent_CountsBySchemeAndReason(t *testing.T) { + r := newPeernetTestRepo(t) + ctx := context.Background() + require.NoError(t, r.UpsertBatch(ctx, []PeernetContentEntry{ + {FilePath: "a", FileID: hexID('a'), ReleaseID: hexID('1'), ReleaseScheme: PeernetSchemeNZB, GroupKey: gk('d')}, + {FilePath: "b", FileID: hexID('b'), ReleaseID: hexID('2'), ReleaseScheme: PeernetSchemeFiles, GroupKey: gk('e')}, + {FilePath: "c", FileID: hexID('c'), ReleaseID: hexID('2'), ReleaseScheme: PeernetSchemeFiles, GroupKey: gk('e')}, + {FilePath: "d", FileID: hexID('d'), GroupKey: gk('f')}, + {FilePath: "e", FileID: hexID('e')}, + })) + c, err := r.Counts(ctx) + require.NoError(t, err) + assert.Equal(t, PeernetContentCounts{Files: 5, WithReleaseID: 3, WithNZBReleaseID: 1, WithDerivedReleaseID: 2, NoSourceGroup: 1}, c) +} + +func TestPeernetContent_GroupSummariesKeysetAndExcludesNZB(t *testing.T) { + r := newPeernetTestRepo(t) + ctx := context.Background() + require.NoError(t, r.UpsertBatch(ctx, []PeernetContentEntry{ + {FilePath: "n/1", FileID: hexID('1'), ReleaseID: hexID('9'), ReleaseScheme: PeernetSchemeNZB, GroupKey: gk('a')}, + {FilePath: "n/2", FileID: hexID('2'), GroupKey: gk('a')}, // old-format sibling of an nzb file + {FilePath: "o/1", FileID: hexID('3'), GroupKey: gk('b')}, + {FilePath: "o/2", FileID: hexID('4'), GroupKey: gk('b')}, + {FilePath: "o/3", FileID: hexID('5'), ReleaseID: hexID('7'), ReleaseScheme: PeernetSchemeFiles, GroupKey: gk('b')}, + {FilePath: "p/1", FileID: hexID('6')}, // no group + })) + first, err := r.GroupSummaries(ctx, "", 1) + require.NoError(t, err) + assert.Equal(t, []PeernetGroupSummary{{Key: gk('a'), Files: 1}}, first) + rest, err := r.GroupSummaries(ctx, first[0].Key, 10) + require.NoError(t, err) + assert.Equal(t, []PeernetGroupSummary{{Key: gk('b'), Files: 3}}, rest) + + members, err := r.GroupMembers(ctx, []string{gk('a'), gk('b')}) + require.NoError(t, err) + require.Len(t, members, 4, "the nzb-scheme file is not a member of a derived group") + assert.Equal(t, "n/2", members[0].FilePath) + assert.Equal(t, "o/1", members[1].FilePath) + none, err := r.GroupMembers(ctx, nil) + require.NoError(t, err) + assert.Empty(t, none) +} + +func TestPeernetContent_ApplyDerivedGuards(t *testing.T) { + r := newPeernetTestRepo(t) + ctx := context.Background() + require.NoError(t, r.UpsertBatch(ctx, []PeernetContentEntry{ + {FilePath: "o/1", FileID: hexID('3'), GroupKey: gk('b')}, + {FilePath: "o/2", FileID: hexID('4'), GroupKey: gk('b')}, + {FilePath: "o/3", FileID: hexID('5'), ReleaseID: hexID('7'), ReleaseScheme: PeernetSchemeNZB, GroupKey: gk('b')}, + })) + n, err := r.ApplyDerived(ctx, []PeernetDerivedUpdate{ + {FilePath: "o/1", FileID: hexID('3'), GroupKey: gk('b'), ReleaseID: hexID('d')}, + {FilePath: "o/2", FileID: hexID('f'), GroupKey: gk('b'), ReleaseID: hexID('d')}, // file changed meanwhile + {FilePath: "o/3", FileID: hexID('5'), GroupKey: gk('b'), ReleaseID: hexID('d')}, // nzb row is never touched + }) + require.NoError(t, err) + assert.Equal(t, 1, n) + got, _, _ := r.GetByPath(ctx, "o/1") + assert.Equal(t, hexID('d'), got.ReleaseID) + assert.Equal(t, PeernetSchemeFiles, got.ReleaseScheme) + two, _, _ := r.GetByPath(ctx, "o/2") + assert.Empty(t, two.ReleaseID) + three, _, _ := r.GetByPath(ctx, "o/3") + assert.Equal(t, hexID('7'), three.ReleaseID) + assert.Equal(t, PeernetSchemeNZB, three.ReleaseScheme) + + // Clearing (empty release ID) returns a row to "no id". + _, err = r.ApplyDerived(ctx, []PeernetDerivedUpdate{{FilePath: "o/1", FileID: hexID('3'), GroupKey: gk('b')}}) + require.NoError(t, err) + got, _, _ = r.GetByPath(ctx, "o/1") + assert.Empty(t, got.ReleaseID) + assert.Equal(t, 0, got.ReleaseScheme) +} + +func TestPeernetContent_StillIndexedAndRefs(t *testing.T) { + r := newPeernetTestRepo(t) + ctx := context.Background() + require.NoError(t, r.UpsertBatch(ctx, []PeernetContentEntry{ + {FilePath: "d/1", FileID: hexID('3'), ReleaseID: hexID('d'), ReleaseScheme: PeernetSchemeFiles, GroupKey: gk('b')}, + {FilePath: "d/sub/2", FileID: hexID('4'), ReleaseID: hexID('d'), ReleaseScheme: PeernetSchemeFiles, GroupKey: gk('b')}, + {FilePath: "e/1", FileID: hexID('5'), ReleaseID: hexID('7'), ReleaseScheme: PeernetSchemeNZB, GroupKey: gk('c')}, + {FilePath: "dd/1", FileID: hexID('6'), GroupKey: gk('9')}, + })) + live, err := r.StillIndexed(ctx, []string{hexID('d'), hexID('e')}) + require.NoError(t, err) + assert.Equal(t, map[string]bool{hexID('d'): true}, live) + + refs, err := r.RefsByPrefix(ctx, "d", 100) + require.NoError(t, err) + assert.Len(t, refs, 2, "exact directory match: dd is not under d") + ref, ok, err := r.RefByPath(ctx, "e/1") + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, PeernetContentRef{GroupKey: gk('c'), ReleaseID: hexID('7'), ReleaseScheme: PeernetSchemeNZB}, ref) + + _, err = r.RefsByPrefix(ctx, "", 10) + assert.Error(t, err) +} + +func TestPeernetContent_VideoFilesOfReleases(t *testing.T) { + r := newPeernetTestRepo(t) + ctx := context.Background() + var rows []PeernetContentEntry + for i := range 20 { + rows = append(rows, PeernetContentEntry{FilePath: fmt.Sprintf("big/e%02d.MKV", i), FileID: hexID(byte('a' + i%6)), ReleaseID: hexID('1'), ReleaseScheme: PeernetSchemeNZB}) + } + rows = append(rows, + PeernetContentEntry{FilePath: "small/m.mkv", FileID: hexID('b'), ReleaseID: hexID('2'), ReleaseScheme: PeernetSchemeFiles}, + PeernetContentEntry{FilePath: "small/m.nfo", FileID: hexID('c'), ReleaseID: hexID('2'), ReleaseScheme: PeernetSchemeFiles}, + PeernetContentEntry{FilePath: "arch/x.r00", FileID: hexID('d'), ReleaseID: hexID('3'), ReleaseScheme: PeernetSchemeFiles}, + ) + require.NoError(t, r.UpsertBatch(ctx, rows)) + got, err := r.VideoFilesOfReleases(ctx, []string{hexID('1'), hexID('2'), hexID('3')}, 16) + require.NoError(t, err) + assert.Len(t, got[hexID('1')], 17, "one more than the cap so the caller can tell it overflowed") + assert.Equal(t, []string{hexID('b')}, got[hexID('2')]) + assert.Empty(t, got[hexID('3')], "a release with no video file has no label files") + + many := make([]string, PeernetMaxPathsPerQuery+1) + for i := range many { + many[i] = fmt.Sprintf("%064x", i+1) + } + _, err = r.VideoFilesOfReleases(ctx, many, 16) + assert.ErrorIs(t, err, ErrPeernetTooManyIDs) +} + +func TestPeernetContent_GroupCountsAndClearDerivedGroup(t *testing.T) { + r := newPeernetTestRepo(t) + ctx := context.Background() + var rows []PeernetContentEntry + for i := range 25 { + rows = append(rows, PeernetContentEntry{FilePath: fmt.Sprintf("big/%02d.mkv", i), FileID: fmt.Sprintf("%064x", i+1), + ReleaseID: hexID('d'), ReleaseScheme: PeernetSchemeFiles, GroupKey: gk('b')}) + } + rows = append(rows, + PeernetContentEntry{FilePath: "n/1", FileID: hexID('1'), ReleaseID: hexID('7'), ReleaseScheme: PeernetSchemeNZB, GroupKey: gk('b')}, + PeernetContentEntry{FilePath: "other/1", FileID: hexID('2'), ReleaseID: hexID('e'), ReleaseScheme: PeernetSchemeFiles, GroupKey: gk('c')}) + require.NoError(t, r.UpsertBatch(ctx, rows)) + + counts, err := r.GroupCounts(ctx, []string{gk('b'), gk('c'), gk('a')}) + require.NoError(t, err) + assert.Equal(t, map[string]int{gk('b'): 25, gk('c'): 1}, counts, "nzb-scheme rows are not counted; empty groups are absent") + + old, err := r.ClearDerivedGroup(ctx, gk('b'), 10) // several short transactions + require.NoError(t, err) + assert.Equal(t, []string{hexID('d')}, old) + left, _ := r.FindByReleaseID(ctx, hexID('d')) + assert.Empty(t, left) + nzb, _, _ := r.GetByPath(ctx, "n/1") + assert.Equal(t, hexID('7'), nzb.ReleaseID, "an nzb-scheme row is never cleared") + other, _, _ := r.GetByPath(ctx, "other/1") + assert.Equal(t, hexID('e'), other.ReleaseID, "another group is untouched") +} diff --git a/internal/database/peernet_content_repository.go b/internal/database/peernet_content_repository.go index 79875897a..babe44552 100644 --- a/internal/database/peernet_content_repository.go +++ b/internal/database/peernet_content_repository.go @@ -22,6 +22,45 @@ type PeernetContentEntry struct { FilePath string FileID string ReleaseID string + // ReleaseScheme says how ReleaseID was derived: PeernetSchemeNone (no + // release ID), PeernetSchemeNZB or PeernetSchemeFiles. A ReleaseID given + // with no scheme is stored as PeernetSchemeNZB. + ReleaseScheme int + // GroupKey is a local-only 64-hex key shared by the files of one import + // (empty when the metadata names no source). It never leaves the node. + GroupKey string +} + +// Release ID schemes stored in peernet_content_index.release_scheme and in +// peernet_records.release_scheme. +const ( + PeernetSchemeNone = 0 + PeernetSchemeNZB = 1 + PeernetSchemeFiles = 2 +) + +// PeernetGroupSummary is one import group and how many of its files still +// need a derived release ID decision (files that are not nzb-scheme). +type PeernetGroupSummary struct { + Key string + Files int +} + +// PeernetDerivedUpdate sets (or, with an empty ReleaseID, clears) the derived +// release ID of one row. FileID and GroupKey guard against a row that changed +// since the group was read. +type PeernetDerivedUpdate struct { + FilePath string + FileID string + GroupKey string + ReleaseID string +} + +// PeernetContentRef is the group and release identity of one row. +type PeernetContentRef struct { + GroupKey string + ReleaseID string + ReleaseScheme int } // PeernetContentRepository stores the local index from virtual file path to @@ -39,14 +78,25 @@ func NewPeernetContentRepository(conn *sql.DB, d Dialect) *PeernetContentReposit } } +// peernetContentCols is the column list every row read uses; scanContent reads it. +const peernetContentCols = `file_path, file_id, release_id, release_scheme, group_key` + const peernetUpsertSQL = ` - INSERT INTO peernet_content_index (file_path, file_id, release_id, indexed_at) - VALUES (?, ?, ?, CURRENT_TIMESTAMP) + INSERT INTO peernet_content_index (file_path, file_id, release_id, release_scheme, group_key, indexed_at) + VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP) ON CONFLICT (file_path) DO UPDATE SET file_id = excluded.file_id, release_id = excluded.release_id, + release_scheme = excluded.release_scheme, + group_key = excluded.group_key, indexed_at = CURRENT_TIMESTAMP` +func scanContent(r rowScanner) (PeernetContentEntry, error) { + var e PeernetContentEntry + err := r.Scan(&e.FilePath, &e.FileID, &e.ReleaseID, &e.ReleaseScheme, &e.GroupKey) + return e, err +} + // normalizePeernetPath folds a virtual path to the form used as the row key: // forward slashes, cleaned, no leading slash. func normalizePeernetPath(p string) string { @@ -74,8 +124,20 @@ func (e PeernetContentEntry) prepare() (PeernetContentEntry, error) { if e.ReleaseID != "" && !isHexID(e.ReleaseID) { return e, fmt.Errorf("peernet content: invalid release id for %q", e.FilePath) } + if e.ReleaseID != "" && e.ReleaseScheme == PeernetSchemeNone { + e.ReleaseScheme = PeernetSchemeNZB // a release ID with no scheme is the original kind + } + switch { + case e.ReleaseScheme < PeernetSchemeNone || e.ReleaseScheme > PeernetSchemeFiles: + return e, fmt.Errorf("peernet content: invalid release scheme for %q", e.FilePath) + case (e.ReleaseID == "") != (e.ReleaseScheme == PeernetSchemeNone): + return e, fmt.Errorf("peernet content: release id and scheme disagree for %q", e.FilePath) + case e.GroupKey != "" && !isHexID(e.GroupKey): + return e, fmt.Errorf("peernet content: invalid group key for %q", e.FilePath) + } e.FileID = strings.ToLower(e.FileID) e.ReleaseID = strings.ToLower(e.ReleaseID) + e.GroupKey = strings.ToLower(e.GroupKey) return e, nil } @@ -85,7 +147,7 @@ func (r *PeernetContentRepository) Upsert(ctx context.Context, e PeernetContentE if err != nil { return err } - if _, err := r.db.ExecContext(ctx, peernetUpsertSQL, e.FilePath, e.FileID, e.ReleaseID); err != nil { + if _, err := r.db.ExecContext(ctx, peernetUpsertSQL, e.FilePath, e.FileID, e.ReleaseID, e.ReleaseScheme, e.GroupKey); err != nil { return fmt.Errorf("upsert peernet content %q: %w", e.FilePath, err) } return nil @@ -118,7 +180,7 @@ func (r *PeernetContentRepository) UpsertBatch(ctx context.Context, entries []Pe } defer stmt.Close() for _, e := range prepared { - if _, err := stmt.ExecContext(ctx, e.FilePath, e.FileID, e.ReleaseID); err != nil { + if _, err := stmt.ExecContext(ctx, e.FilePath, e.FileID, e.ReleaseID, e.ReleaseScheme, e.GroupKey); err != nil { return fmt.Errorf("upsert peernet content %q: %w", e.FilePath, err) } } @@ -135,10 +197,8 @@ func (r *PeernetContentRepository) GetByPath(ctx context.Context, filePath strin if p == "" { return PeernetContentEntry{}, false, errors.New("peernet content: empty file path") } - var e PeernetContentEntry - err := r.db.QueryRowContext(ctx, - `SELECT file_path, file_id, release_id FROM peernet_content_index WHERE file_path = ?`, p). - Scan(&e.FilePath, &e.FileID, &e.ReleaseID) + e, err := scanContent(r.db.QueryRowContext(ctx, + `SELECT `+peernetContentCols+` FROM peernet_content_index WHERE file_path = ?`, p)) if errors.Is(err, sql.ErrNoRows) { return PeernetContentEntry{}, false, nil } @@ -209,15 +269,15 @@ func (r *PeernetContentRepository) FindByReleaseID(ctx context.Context, releaseI func (r *PeernetContentRepository) find(ctx context.Context, column, id string) ([]PeernetContentEntry, error) { // column is one of two compile-time constants, never caller input. rows, err := r.db.QueryContext(ctx, - `SELECT file_path, file_id, release_id FROM peernet_content_index WHERE `+column+` = ? ORDER BY file_path`, id) + `SELECT `+peernetContentCols+` FROM peernet_content_index WHERE `+column+` = ? ORDER BY file_path`, id) if err != nil { return nil, fmt.Errorf("query peernet content by %s: %w", column, err) } defer rows.Close() var out []PeernetContentEntry for rows.Next() { - var e PeernetContentEntry - if err := rows.Scan(&e.FilePath, &e.FileID, &e.ReleaseID); err != nil { + e, err := scanContent(rows) + if err != nil { return nil, fmt.Errorf("scan peernet content: %w", err) } out = append(out, e) @@ -302,7 +362,7 @@ func (r *PeernetContentRepository) ListPage(ctx context.Context, after string, l return nil, errors.New("peernet content: page limit must be positive") } rows, err := r.db.QueryContext(ctx, - `SELECT file_path, file_id, release_id FROM peernet_content_index + `SELECT `+peernetContentCols+` FROM peernet_content_index WHERE file_path > ? ORDER BY file_path LIMIT ?`, after, limit) if err != nil { return nil, fmt.Errorf("list peernet content page: %w", err) @@ -310,8 +370,8 @@ func (r *PeernetContentRepository) ListPage(ctx context.Context, after string, l defer rows.Close() var out []PeernetContentEntry for rows.Next() { - var e PeernetContentEntry - if err := rows.Scan(&e.FilePath, &e.FileID, &e.ReleaseID); err != nil { + e, err := scanContent(rows) + if err != nil { return nil, fmt.Errorf("scan peernet content: %w", err) } out = append(out, e) @@ -374,17 +434,25 @@ func (r *PeernetContentRepository) MissingPaths(ctx context.Context, candidates } // PeernetContentCounts summarises the index: every indexed file and how many -// of them have a Release ID. +// of them have a Release ID, by scheme. NoSourceGroup counts files with no +// Release ID whose metadata names no source (they can never be grouped). type PeernetContentCounts struct { - Files, WithReleaseID int + Files, WithReleaseID int + WithNZBReleaseID, WithDerivedReleaseID int + NoSourceGroup int } // Counts returns the size of the index. func (r *PeernetContentRepository) Counts(ctx context.Context) (PeernetContentCounts, error) { var c PeernetContentCounts err := r.db.QueryRowContext(ctx, - `SELECT COUNT(*), COALESCE(SUM(CASE WHEN release_id <> '' THEN 1 ELSE 0 END), 0) FROM peernet_content_index`). - Scan(&c.Files, &c.WithReleaseID) + `SELECT COUNT(*), + COALESCE(SUM(CASE WHEN release_id <> '' THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN release_scheme = 1 THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN release_scheme = 2 THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN release_id = '' AND group_key = '' THEN 1 ELSE 0 END), 0) + FROM peernet_content_index`). + Scan(&c.Files, &c.WithReleaseID, &c.WithNZBReleaseID, &c.WithDerivedReleaseID, &c.NoSourceGroup) if err != nil { return PeernetContentCounts{}, fmt.Errorf("count peernet content: %w", err) } diff --git a/internal/database/peernet_content_repository_test.go b/internal/database/peernet_content_repository_test.go index c7a2eeac3..017f57b23 100644 --- a/internal/database/peernet_content_repository_test.go +++ b/internal/database/peernet_content_repository_test.go @@ -33,7 +33,7 @@ func TestPeernetContentRepository_UpsertAndLookup(t *testing.T) { byFile, err := r.FindByFileID(ctx, hexID('a')) require.NoError(t, err) - assert.Equal(t, []PeernetContentEntry{{FilePath: "movies/a.mkv", FileID: hexID('a'), ReleaseID: hexID('1')}}, byFile) + assert.Equal(t, []PeernetContentEntry{{FilePath: "movies/a.mkv", FileID: hexID('a'), ReleaseID: hexID('1'), ReleaseScheme: PeernetSchemeNZB}}, byFile) byRel, err := r.FindByReleaseID(ctx, hexID('1')) require.NoError(t, err) @@ -173,7 +173,7 @@ func TestPeernetContentRepository_GetByPath(t *testing.T) { got, ok, err := r.GetByPath(ctx, p) require.NoError(t, err) require.True(t, ok, p) - assert.Equal(t, PeernetContentEntry{FilePath: "movies/a.mkv", FileID: hexID('a'), ReleaseID: hexID('1')}, got) + assert.Equal(t, PeernetContentEntry{FilePath: "movies/a.mkv", FileID: hexID('a'), ReleaseID: hexID('1'), ReleaseScheme: PeernetSchemeNZB}, got) } _, _, err = r.GetByPath(ctx, "") assert.Error(t, err) @@ -213,5 +213,5 @@ func TestPeernetContentRepository_Counts(t *testing.T) { require.NoError(t, r.Upsert(ctx, PeernetContentEntry{FilePath: "c.mkv", FileID: hexID('c')})) c, err = r.Counts(ctx) require.NoError(t, err) - assert.Equal(t, PeernetContentCounts{Files: 3, WithReleaseID: 1}, c) + assert.Equal(t, PeernetContentCounts{Files: 3, WithReleaseID: 1, WithNZBReleaseID: 1, NoSourceGroup: 2}, c) } diff --git a/internal/database/peernet_password_origin_repository.go b/internal/database/peernet_password_origin_repository.go new file mode 100644 index 000000000..31b4a4951 --- /dev/null +++ b/internal/database/peernet_password_origin_repository.go @@ -0,0 +1,56 @@ +package database + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" +) + +// PeernetPasswordOriginRepository stores which files were imported with a +// password that came with the NZB's own content (table peernet_password_origin). +type PeernetPasswordOriginRepository struct{ db *dialectAwareDB } + +// NewPeernetPasswordOriginRepository creates a repository over conn. +func NewPeernetPasswordOriginRepository(conn *sql.DB, d Dialect) *PeernetPasswordOriginRepository { + return &PeernetPasswordOriginRepository{db: newDialectAwareDB(conn, d)} +} + +// MarkFromNZB records that the files with these File IDs were imported with a +// password that came with the NZB's own content. It is idempotent. An ID that +// is not a well-formed File ID is an error and nothing is written. +func (r *PeernetPasswordOriginRepository) MarkFromNZB(ctx context.Context, fileIDs []string) error { + for _, id := range fileIDs { + if !isHexID(id) { + return errors.New("peernet password origin: bad file id") + } + } + now := time.Now().Unix() + for _, id := range fileIDs { + if _, err := r.db.ExecContext(ctx, `INSERT INTO peernet_password_origin (file_id, recorded_at) VALUES (?, ?) + ON CONFLICT(file_id) DO NOTHING`, strings.ToLower(id), now); err != nil { + return fmt.Errorf("peernet password origin: mark: %w", err) + } + } + return nil +} + +// FromNZB reports whether the file with this File ID was imported with a +// password that came with the NZB's own content. Anything unrecorded, and any +// malformed ID, is false. +func (r *PeernetPasswordOriginRepository) FromNZB(ctx context.Context, fileID string) (bool, error) { + if !isHexID(fileID) { + return false, nil + } + var one int + err := r.db.QueryRowContext(ctx, `SELECT 1 FROM peernet_password_origin WHERE file_id = ?`, strings.ToLower(fileID)).Scan(&one) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("peernet password origin: read: %w", err) + } + return true, nil +} diff --git a/internal/database/peernet_password_origin_repository_test.go b/internal/database/peernet_password_origin_repository_test.go new file mode 100644 index 000000000..cc61d1bab --- /dev/null +++ b/internal/database/peernet_password_origin_repository_test.go @@ -0,0 +1,45 @@ +package database + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPeernetPasswordOrigin(t *testing.T) { + db, err := NewDB(Config{DatabasePath: filepath.Join(t.TempDir(), "db")}) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + run := func(t *testing.T, r *PeernetPasswordOriginRepository) { + ctx := context.Background() + ok, err := r.FromNZB(ctx, hex64("a")) + require.NoError(t, err) + assert.False(t, ok, "no row: not from the NZB") + + require.NoError(t, r.MarkFromNZB(ctx, []string{hex64("a"), hex64("b")})) + require.NoError(t, r.MarkFromNZB(ctx, []string{hex64("a")}), "idempotent") + require.NoError(t, r.MarkFromNZB(ctx, nil), "nothing to do") + for _, id := range []string{hex64("a"), hex64("b")} { + ok, err = r.FromNZB(ctx, id) + require.NoError(t, err) + assert.True(t, ok) + } + ok, err = r.FromNZB(ctx, hex64("c")) + require.NoError(t, err) + assert.False(t, ok) + + assert.Error(t, r.MarkFromNZB(ctx, []string{"not-an-id"}), "only well-formed File IDs are stored") + ok, err = r.FromNZB(ctx, "not-an-id") + require.NoError(t, err) + assert.False(t, ok) + } + t.Run("sqlite", func(t *testing.T) { run(t, NewPeernetPasswordOriginRepository(db.Connection(), DialectSQLite)) }) + if hasPostgresDSN() { + t.Run("postgres", func(t *testing.T) { + run(t, NewPeernetPasswordOriginRepository(openPostgresLatest(t), DialectPostgres)) + }) + } +} diff --git a/internal/database/peernet_publish_state_repository.go b/internal/database/peernet_publish_state_repository.go index 423a206b2..6143b0156 100644 --- a/internal/database/peernet_publish_state_repository.go +++ b/internal/database/peernet_publish_state_repository.go @@ -24,6 +24,13 @@ type PeernetPublishState struct { // (lower-cased, trimmed) of the release's origin at publish time. OriginCategory string OriginIndexer string + // BundleID is the Bundle ID announced in the release record (lower-case + // hex; empty when none). BundleLevel is the exposure level the record was + // published at; at 3 or more the record was published with the bundle + // decision made (an empty BundleID then means the release is not + // shareable as a bundle). Release records only. + BundleID string + BundleLevel int } // PeernetPublishStateRepository stores last-published state (table @@ -59,9 +66,10 @@ func (r *PeernetPublishStateRepository) Get(ctx context.Context, network string, s := PeernetPublishState{NetworkID: network, Kind: kind, ReleaseID: release, FileID: file} var published, checked int64 err := r.db.QueryRowContext(ctx, - `SELECT published_at, checked_at, verdict, origin_category, origin_indexer FROM peernet_publish_state - WHERE network_id = ? AND kind = ? AND release_id = ? AND file_id = ?`, - network, kind, release, file).Scan(&published, &checked, &s.Verdict, &s.OriginCategory, &s.OriginIndexer) + `SELECT published_at, checked_at, verdict, origin_category, origin_indexer, bundle_id, bundle_level + FROM peernet_publish_state WHERE network_id = ? AND kind = ? AND release_id = ? AND file_id = ?`, + network, kind, release, file).Scan(&published, &checked, &s.Verdict, &s.OriginCategory, &s.OriginIndexer, + &s.BundleID, &s.BundleLevel) if errors.Is(err, sql.ErrNoRows) { return PeernetPublishState{}, false, nil } @@ -77,15 +85,19 @@ func (r *PeernetPublishStateRepository) Put(ctx context.Context, s PeernetPublis if err := validStateKey(s.NetworkID, s.Kind, s.ReleaseID, s.FileID); err != nil { return err } + if s.BundleID != "" && (s.Kind != PeernetRecordKindRelease || !isLowerHexID(s.BundleID)) { + return errors.New("peernet publish state: invalid bundle id") + } _, err := r.db.ExecContext(ctx, `INSERT INTO peernet_publish_state (network_id, kind, release_id, file_id, published_at, checked_at, verdict, - origin_category, origin_indexer) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + origin_category, origin_indexer, bundle_id, bundle_level) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (network_id, kind, release_id, file_id) DO UPDATE SET published_at = excluded.published_at, checked_at = excluded.checked_at, verdict = excluded.verdict, - origin_category = excluded.origin_category, origin_indexer = excluded.origin_indexer`, + origin_category = excluded.origin_category, origin_indexer = excluded.origin_indexer, + bundle_id = excluded.bundle_id, bundle_level = excluded.bundle_level`, s.NetworkID, s.Kind, s.ReleaseID, s.FileID, unixOrZero(s.PublishedAt), unixOrZero(s.CheckedAt), s.Verdict, - normTag(s.OriginCategory), normTag(s.OriginIndexer)) + normTag(s.OriginCategory), normTag(s.OriginIndexer), s.BundleID, s.BundleLevel) if err != nil { return fmt.Errorf("put peernet publish state: %w", err) } diff --git a/internal/database/peernet_queue_hold_test.go b/internal/database/peernet_queue_hold_test.go new file mode 100644 index 000000000..f33e7e403 --- /dev/null +++ b/internal/database/peernet_queue_hold_test.go @@ -0,0 +1,192 @@ +package database + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// holdFixture is a completed queue item of a peer import, with its provisional +// history row, on one backend. +type holdFixture struct { + repo *Repository + bund *PeernetBundleImportRepository + id int64 +} + +func newHoldFixture(t *testing.T, r *PeernetBundleImportRepository, name string, skip bool) holdFixture { + t.Helper() + ctx := context.Background() + repo := NewRepository(r.db.db, r.db.dialect.d) + cat := "tv" + size := int64(100) + item := &ImportQueueItem{NzbPath: name + ".nzb", Category: &cat, Priority: QueuePriorityNormal, + Status: QueueStatusProcessing, MaxRetries: 1, FileSize: &size, SkipArrNotification: skip} + require.NoError(t, repo.AddToQueue(ctx, item)) + nzb := item.ID + require.NoError(t, repo.AddImportHistory(ctx, &ImportHistory{NzbID: &nzb, NzbName: name + ".nzb", FileName: "a.mkv", + FileSize: size, VirtualPath: "tv/" + name, Category: &cat})) + require.NoError(t, repo.UpdateQueueItemStatus(ctx, item.ID, QueueStatusCompleted, nil)) + return holdFixture{repo: repo, bund: r, id: item.ID} +} + +func (f holdFixture) historyStatuses(t *testing.T) map[int64]QueueStatus { + t.Helper() + rows, err := f.repo.ListSABnzbdHistory(context.Background(), "", 100, 0) + require.NoError(t, err) + out := map[int64]QueueStatus{} + for _, r := range rows { + out[r.ID] = r.Status + } + n, err := f.repo.CountSABnzbdHistory(context.Background(), "") + require.NoError(t, err) + require.Equal(t, len(out), n) + return out +} + +func (f holdFixture) activeIDs(t *testing.T, withHeld bool) map[int64]bool { + t.Helper() + list := f.repo.ListActiveQueueItems + count := f.repo.CountActiveQueueItems + if withHeld { + list, count = f.repo.ListActiveQueueItemsWithHeld, f.repo.CountActiveQueueItemsWithHeld + } + items, err := list(context.Background(), "", "", 100, 0, "updated_at", "desc") + require.NoError(t, err) + n, err := count(context.Background(), "", "") + require.NoError(t, err) + require.Equal(t, len(items), n) + out := map[int64]bool{} + for _, it := range items { + out[it.ID] = it.PeerHold + } + return out +} + +func TestPeerHold_PendingIsInQueueNotHistory(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + f := newHoldFixture(t, r, "pending", false) + require.Contains(t, f.historyStatuses(t), f.id, "an ordinary completed item is in the history") + + require.NoError(t, r.SetQueueHold(ctx, f.id, true)) + assert.NotContains(t, f.historyStatuses(t), f.id, "a held item must not be in the history") + assert.Equal(t, map[int64]bool{f.id: true}, f.activeIDs(t, true), "a held item is in the queue, marked held") + assert.Empty(t, f.activeIDs(t, false), "the plain active list is unchanged") + got, err := f.repo.GetQueueItem(ctx, f.id) + require.NoError(t, err) + assert.True(t, got.PeerHold) + assert.False(t, got.SkipArrNotification, "the hold is not the user's skip flag") + }) +} + +func TestPeerHold_PassedMovesToHistory(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + f := newHoldFixture(t, r, "passed", false) + require.NoError(t, r.SetQueueHold(ctx, f.id, true)) + require.NoError(t, r.SetQueueHold(ctx, f.id, false)) + assert.Equal(t, QueueStatusCompleted, f.historyStatuses(t)[f.id]) + assert.Empty(t, f.activeIDs(t, true)) + }) +} + +func TestPeerHold_FailedIsFailedInHistoryWithMessage(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + f := newHoldFixture(t, r, "failed", false) + require.NoError(t, r.SetQueueHold(ctx, f.id, true)) + changed, err := r.FailHeldQueueItem(ctx, f.id, "peer import failed its full check") + require.NoError(t, err) + require.True(t, changed) + assert.Equal(t, QueueStatusFailed, f.historyStatuses(t)[f.id]) + rows, err := f.repo.ListSABnzbdFailedHistory(ctx, "", 10) + require.NoError(t, err) + require.Len(t, rows, 1) + require.NotNil(t, rows[0].ErrorMessage) + assert.Equal(t, "peer import failed its full check", *rows[0].ErrorMessage) + assert.Empty(t, f.activeIDs(t, true)) + + // The provisional history row is gone: when the retention cleanup later + // deletes the failed queue row, nothing resurfaces as completed. + require.NoError(t, f.repo.RemoveFromQueue(ctx, f.id)) + assert.NotContains(t, f.historyStatuses(t), f.id) + + changed, err = r.FailHeldQueueItem(ctx, f.id, "again") + require.NoError(t, err) + assert.False(t, changed, "only a held item can be failed") + }) +} + +func TestPeerHold_UserSkipStaysHidden(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + f := newHoldFixture(t, r, "skip", true) + require.NoError(t, r.SetQueueHold(ctx, f.id, true)) + require.NoError(t, r.SetQueueHold(ctx, f.id, false)) + assert.NotContains(t, f.historyStatuses(t), f.id, "the user's skip is still honoured after the hold ends") + + require.NoError(t, r.SetQueueHold(ctx, f.id, true)) + changed, err := r.FailHeldQueueItem(ctx, f.id, "x") + require.NoError(t, err) + require.True(t, changed) + assert.NotContains(t, f.historyStatuses(t), f.id, "a failed skipped item stays hidden") + }) +} + +func TestPeerBundleImport_CommitMarkerAndClaimGuard(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + require.NoError(t, r.Upsert(ctx, []PeernetBundleImport{bundleRow("m/a.mkv", 5), bundleRow("m/b.mkv", 5)})) + _, err := r.MarkPassed(ctx, "m/a.mkv") + require.NoError(t, err) + _, err = r.MarkPassed(ctx, "m/b.mkv") + require.NoError(t, err) + + un, err := r.ListUncommitted(ctx, 10) + require.NoError(t, err) + assert.Len(t, un, 2, "rows start uncommitted") + _, claimed, err := r.ClaimAnnouncement(ctx, 5) + require.NoError(t, err) + assert.False(t, claimed, "an unfinished write is never announced") + ids, err := r.QueuesAwaitingAnnouncement(ctx) + require.NoError(t, err) + assert.Empty(t, ids) + + require.NoError(t, r.CommitRows(ctx, []string{"/m/a.mkv", "m/b.mkv"})) + un, err = r.ListUncommitted(ctx, 10) + require.NoError(t, err) + assert.Empty(t, un) + byQ, err := r.ListByQueue(ctx, 5) + require.NoError(t, err) + require.Len(t, byQ, 2) + assert.True(t, byQ[0].Committed) + _, claimed, err = r.ClaimAnnouncement(ctx, 5) + require.NoError(t, err) + assert.True(t, claimed) + + page, err := r.ListAfter(ctx, "m/a.mkv", 10) + require.NoError(t, err) + require.Len(t, page, 1) + assert.Equal(t, "m/b.mkv", page[0].FilePath) + }) +} + +func TestPeerBundleImport_OrphanedHolds(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + referenced := newHoldFixture(t, r, "referenced", false) + orphan := newHoldFixture(t, r, "orphan", false) + plain := newHoldFixture(t, r, "plain", false) // never held + require.NoError(t, r.SetQueueHold(ctx, referenced.id, true)) + require.NoError(t, r.SetQueueHold(ctx, orphan.id, true)) + require.NoError(t, r.Upsert(ctx, []PeernetBundleImport{bundleRow("m/a.mkv", referenced.id)})) + + got, err := r.OrphanedHolds(ctx) + require.NoError(t, err) + assert.Equal(t, []int64{orphan.id}, got, "only a held item nothing refers to") + _ = plain + }) +} diff --git a/internal/database/peernet_record_files_test.go b/internal/database/peernet_record_files_test.go new file mode 100644 index 000000000..4b57a8647 --- /dev/null +++ b/internal/database/peernet_record_files_test.go @@ -0,0 +1,158 @@ +package database + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPeernetRecords_AttestationsForFilesMatchesAcrossReleaseIDs(t *testing.T) { + forEachRecBackend(t, func(t *testing.T, b *recBackend) { + ctx := context.Background() + file := fileHex(5) + // The same file attested under an "nzb" and a "files" release ID. + insertOK(t, b, mkAtt(true, func(r *PeernetRecord) { r.ReleaseID = relHex(1); r.FileID = file; r.PublisherNodeID = nodeHex(1) })) + insertOK(t, b, mkAtt(true, func(r *PeernetRecord) { + r.ReleaseID, r.FileID, r.PublisherNodeID, r.ReleaseScheme = relHex(2), file, nodeHex(2), PeernetSchemeFiles + })) + insertOK(t, b, mkAtt(true, func(r *PeernetRecord) { r.ReleaseID = relHex(3); r.FileID = fileHex(6); r.PublisherNodeID = nodeHex(3) })) + insertOK(t, b, mkRec(func(r *PeernetRecord) { r.ReleaseID = relHex(1); r.PublisherNodeID = nodeHex(4) })) + + got, err := b.recs.AttestationsForFiles(ctx, hexID('a'), []string{file}, time.Time{}) + require.NoError(t, err) + require.Len(t, got, 2, "both publishers, whatever release ID they used; no other file, no release record") + pubs := []string{got[0].PublisherNodeID, got[1].PublisherNodeID} + assert.ElementsMatch(t, []string{nodeHex(1), nodeHex(2)}, pubs) + for _, a := range got { + assert.Equal(t, file, a.FileID) + } + + other, err := b.recs.AttestationsForFiles(ctx, hexID('b'), []string{file}, time.Time{}) + require.NoError(t, err) + assert.Empty(t, other, "networks stay separate") + + many, err := b.recs.AttestationsForFiles(ctx, hexID('a'), []string{file, fileHex(6), file}, time.Time{}) + require.NoError(t, err) + assert.Len(t, many, 3, "one batch, duplicates in the request are ignored") + }) +} + +// One publisher must not be able to occupy the result with rows it made up +// under many release IDs for the same file. +func TestPeernetRecords_AttestationsForFilesOneRowPerPublisherAndVerdict(t *testing.T) { + forEachRecBackend(t, func(t *testing.T, b *recBackend) { + ctx := context.Background() + file := fileHex(9) + for i := range 40 { + insertOK(t, b, mkAtt(true, func(r *PeernetRecord) { + r.ReleaseID, r.FileID, r.PublisherNodeID = relHex(100+i), file, nodeHex(1) + r.CheckedAt = recT0.Add(-time.Duration(i+1) * time.Minute) + })) + } + insertOK(t, b, mkAtt(false, func(r *PeernetRecord) { r.ReleaseID, r.FileID, r.PublisherNodeID = relHex(500), file, nodeHex(1) })) + insertOK(t, b, mkAtt(true, func(r *PeernetRecord) { r.ReleaseID, r.FileID, r.PublisherNodeID = relHex(501), file, nodeHex(2) })) + + got, err := b.recs.AttestationsForFiles(ctx, hexID('a'), []string{file}, time.Time{}) + require.NoError(t, err) + require.Len(t, got, 3, "publisher 1 healthy (newest only), publisher 1 unhealthy, publisher 2 healthy") + assert.False(t, got[0].Healthy, "unhealthy rows come first so truncation can only drop healthy evidence") + var p1Healthy PeernetAttestation + for _, a := range got { + if a.PublisherNodeID == nodeHex(1) && a.Healthy { + p1Healthy = a + } + } + assert.Equal(t, recT0.Add(-time.Minute).Unix(), p1Healthy.CheckedAt.Unix(), "the newest of the publisher's rows is kept") + }) +} + +func TestPeernetRecords_AttestationsForFilesFiltersAndBounds(t *testing.T) { + forEachRecBackend(t, func(t *testing.T, b *recBackend) { + ctx := context.Background() + file := fileHex(7) + insertOK(t, b, mkAtt(true, func(r *PeernetRecord) { + r.FileID, r.PublisherNodeID = file, nodeHex(1) + r.CheckedAt = recT0.Add(-10 * 24 * time.Hour) + r.CreatedAt = recT0.Add(-10 * 24 * time.Hour) + r.ExpiresAt = recT0.Add(time.Hour) + })) + insertOK(t, b, mkAtt(true, func(r *PeernetRecord) { r.FileID, r.PublisherNodeID = file, nodeHex(2) })) + got, err := b.recs.AttestationsForFiles(ctx, hexID('a'), []string{file}, recT0.Add(-7*24*time.Hour)) + require.NoError(t, err) + require.Len(t, got, 1, "stale check excluded") + + b.setNow(recT0.Add(30 * time.Hour)) + none, err := b.recs.AttestationsForFiles(ctx, hexID('a'), []string{file}, time.Time{}) + require.NoError(t, err) + assert.Empty(t, none, "expired rows are not evidence") + + empty, err := b.recs.AttestationsForFiles(ctx, hexID('a'), nil, time.Time{}) + require.NoError(t, err) + assert.Empty(t, empty) + + ids := make([]string, PeernetMaxFilesPerQuery+1) + for i := range ids { + ids[i] = fileHex(1000 + i) + } + _, err = b.recs.AttestationsForFiles(ctx, hexID('a'), ids, time.Time{}) + assert.ErrorIs(t, err, ErrPeernetTooManyIDs) + _, err = b.recs.AttestationsForFiles(ctx, hexID('a'), ids[:PeernetMaxFilesPerQuery], time.Time{}) + assert.NoError(t, err, "exactly at the bound works") + _, err = b.recs.AttestationsForFiles(ctx, "bad", []string{file}, time.Time{}) + assert.Error(t, err) + _, err = b.recs.AttestationsForFiles(ctx, hexID('a'), []string{"nothex"}, time.Time{}) + assert.Error(t, err) + }) +} + +func TestPeernetRecords_ReleaseSchemeStoredAndReturned(t *testing.T) { + forEachRecBackend(t, func(t *testing.T, b *recBackend) { + ctx := context.Background() + insertOK(t, b, mkRec(func(r *PeernetRecord) { r.IMDbID = "tt0000009"; r.ReleaseID = relHex(70) })) + insertOK(t, b, mkRec(func(r *PeernetRecord) { + r.IMDbID = "tt0000009" + r.ReleaseID = relHex(71) + r.ReleaseScheme = PeernetSchemeFiles + r.PublisherNodeID = nodeHex(2) + })) + got, err := b.recs.CandidatesByMedia(ctx, []string{hexID('a')}, "tt0000009", nil, nil, 10) + require.NoError(t, err) + schemes := map[string]int{} + for _, c := range got { + schemes[c.ReleaseID] = c.ReleaseScheme + } + assert.Equal(t, map[string]int{relHex(70): PeernetSchemeNZB, relHex(71): PeernetSchemeFiles}, schemes, + "an unset scheme is stored as nzb") + + bad := mkRec(func(r *PeernetRecord) { r.ReleaseScheme = 3 }) + _, err = b.recs.Insert(ctx, bad, PeernetRecordLimits{}) + assert.ErrorIs(t, err, ErrPeernetRecordInvalid) + }) +} + +// The file lookup must be served by an index, not a table scan. +func TestPeernetRecords_FileLookupUsesIndex(t *testing.T) { + forEachRecBackend(t, func(t *testing.T, b *recBackend) { + if strings.Contains(t.Name(), "postgres") { + t.Skip("plan shape depends on table statistics; checked on sqlite") + } + rows, err := b.recs.db.QueryContext(context.Background(), + "EXPLAIN QUERY PLAN "+attestationsForFilesSQL(2), hexID('a'), 2, fileHex(1), fileHex(2), 0, 0, 10) + require.NoError(t, err) + defer rows.Close() + var plan []string + for rows.Next() { + var id, parent, unused int + var detail string + require.NoError(t, rows.Scan(&id, &parent, &unused, &detail)) + plan = append(plan, detail) + } + joined := strings.Join(plan, "; ") + assert.Contains(t, joined, "idx_peernet_records_file", fmt.Sprintf("plan: %s", joined)) + }) +} diff --git a/internal/database/peernet_record_queries.go b/internal/database/peernet_record_queries.go index bb5999164..de03f3ee5 100644 --- a/internal/database/peernet_record_queries.go +++ b/internal/database/peernet_record_queries.go @@ -302,6 +302,8 @@ type PeernetCandidate struct { TotalSize int64 CreatedAt time.Time Source int + // ReleaseScheme is PeernetSchemeNZB or PeernetSchemeFiles. + ReleaseScheme int } // CandidatesByMedia returns non-expired release records for a media key across @@ -331,7 +333,7 @@ func (r *PeernetRecordRepository) CandidatesByMedia(ctx context.Context, network args = append(args, n) } q := `SELECT seq, network_id, release_id, publisher_node_id, imdb_id, season, episode, title, total_size, - created_at, source + created_at, source, release_scheme FROM peernet_records WHERE network_id IN (?` + strings.Repeat(",?", len(networkIDs)-1) + `) AND kind = ? AND imdb_id = ? AND expires_at > ?` @@ -360,7 +362,7 @@ func (r *PeernetRecordRepository) CandidatesByMedia(ctx context.Context, network var s, e sql.NullInt64 var created int64 if err := rows.Scan(&c.Seq, &c.NetworkID, &c.ReleaseID, &c.PublisherNodeID, &c.IMDbID, &s, &e, &c.Title, - &c.TotalSize, &created, &c.Source); err != nil { + &c.TotalSize, &created, &c.Source, &c.ReleaseScheme); err != nil { return nil, fmt.Errorf("scan peernet candidate: %w", err) } if s.Valid { @@ -432,3 +434,90 @@ func (r *PeernetRecordRepository) Stats(ctx context.Context, networkID string) ( } return out, nil } + +// PeernetMaxFilesPerQuery bounds AttestationsForFiles input. +const PeernetMaxFilesPerQuery = 1024 + +// peernetFileAttestationsMax caps the rows AttestationsForFiles returns. +const peernetFileAttestationsMax = 5000 + +// attestationsForFilesSQL is the file-evidence query for n file IDs. The unary +// plus on expires_at stops SQLite from preferring the (network, expires_at) +// index, which it otherwise picks when it has no statistics: the file index is +// the selective one, and a scan of every live record of a network is exactly +// the cost this query must not have. PostgreSQL ignores the plus. +func attestationsForFilesSQL(n int) string { + return `SELECT publisher_node_id, file_id, healthy, backbones, checked_at, created_at, source, + segments_checked, total_segments + FROM ( + SELECT publisher_node_id, file_id, healthy, backbones, checked_at, created_at, source, + segments_checked, total_segments, + ROW_NUMBER() OVER (PARTITION BY file_id, publisher_node_id, healthy ORDER BY checked_at DESC, seq DESC) AS rn + FROM peernet_records + WHERE network_id = ? AND kind = ? AND file_id IN (?` + strings.Repeat(",?", n-1) + `) + AND +expires_at > ? AND checked_at >= ? AND healthy IS NOT NULL + ) ranked + WHERE rn = 1 + ORDER BY healthy, checked_at DESC, publisher_node_id, file_id + LIMIT ?` +} + +// AttestationsForFiles returns the non-expired attestations, checked at or +// after notBefore, of the given files in a network, whatever release ID they +// were published under: a file's articles are the same whichever release it +// is part of, so evidence about the file applies to every copy of it. +// +// At most one row per (file, publisher, verdict) is returned, the newest, so a +// publisher cannot fill the result by attesting one file under many invented +// release IDs, and a publisher counts once per file. Unhealthy rows come first, +// then newest check first, and at most 5000 rows are returned: truncation can +// only ever drop healthy evidence. At most PeernetMaxFilesPerQuery distinct +// files are accepted (ErrPeernetTooManyIDs beyond that). checked_at is clamped +// to the receive time on insert (see AttestationsFor). +func (r *PeernetRecordRepository) AttestationsForFiles(ctx context.Context, networkID string, fileIDs []string, notBefore time.Time) ([]PeernetAttestation, error) { + if !isLowerHexID(networkID) { + return nil, errors.New("peernet records: invalid network id") + } + seen := make(map[string]struct{}, len(fileIDs)) + ids := make([]string, 0, len(fileIDs)) + for _, id := range fileIDs { + if !isLowerHexID(id) { + return nil, errors.New("peernet records: invalid file id") + } + if _, dup := seen[id]; !dup { + seen[id] = struct{}{} + ids = append(ids, id) + } + } + if len(ids) > PeernetMaxFilesPerQuery { + return nil, ErrPeernetTooManyIDs + } + if len(ids) == 0 { + return nil, nil + } + args := []any{networkID, PeernetRecordKindAttestation} + for _, id := range ids { + args = append(args, id) + } + args = append(args, r.now().Unix(), unixOrZero(notBefore), peernetFileAttestationsMax) + rows, err := r.db.QueryContext(ctx, attestationsForFilesSQL(len(ids)), args...) + if err != nil { + return nil, fmt.Errorf("query peernet attestations by file: %w", err) + } + defer rows.Close() + var out []PeernetAttestation + for rows.Next() { + var a PeernetAttestation + var healthy sql.NullInt64 + var bb string + var checked, created int64 + if err := rows.Scan(&a.PublisherNodeID, &a.FileID, &healthy, &bb, &checked, &created, &a.Source, &a.SegmentsChecked, &a.TotalSegments); err != nil { + return nil, fmt.Errorf("scan peernet attestation: %w", err) + } + a.Healthy = healthy.Valid && healthy.Int64 != 0 + a.Backbones = splitBackbones(bb) + a.CheckedAt, a.CreatedAt = timeOrZero(checked), timeOrZero(created) + out = append(out, a) + } + return out, rows.Err() +} diff --git a/internal/database/peernet_record_repository.go b/internal/database/peernet_record_repository.go index 3adf1f8b4..2ba449489 100644 --- a/internal/database/peernet_record_repository.go +++ b/internal/database/peernet_record_repository.go @@ -10,6 +10,7 @@ import ( "hash/fnv" "sort" "strings" + "sync/atomic" "time" ) @@ -150,6 +151,14 @@ type PeernetRecord struct { Title string TotalSize int64 + // ReleaseScheme says how ReleaseID was derived: PeernetSchemeNZB (the + // default, also for 0) or PeernetSchemeFiles. It comes from the signed + // record. + ReleaseScheme int + // BundleID is the Bundle ID the signed release record announces (lower-case + // hex), empty when the publisher serves no bundle. + BundleID string + Payload []byte } @@ -244,6 +253,12 @@ func (r PeernetRecord) validate(now time.Time) error { return invalid("relayer node id") case r.Source == PeernetRecordSourceLocal && r.RelayerNodeID != "": return invalid("own record with a relayer") + case r.ReleaseScheme < 0 || r.ReleaseScheme > PeernetSchemeFiles: + return invalid("release scheme %d", r.ReleaseScheme) + case r.BundleID != "" && !isLowerHexID(r.BundleID): + return invalid("bundle id") + case r.BundleID != "" && r.Kind != PeernetRecordKindRelease: + return invalid("bundle id on a non-release record") case r.SegmentsChecked < 0 || r.TotalSegments < 0: return invalid("negative segment count") case len(r.Payload) == 0: @@ -424,6 +439,10 @@ func (r *PeernetRecordRepository) insert(ctx context.Context, rec PeernetRecord, if rec.Healthy != nil { healthy = boolInt(*rec.Healthy) } + scheme := rec.ReleaseScheme + if scheme == PeernetSchemeNone { + scheme = PeernetSchemeNZB + } var season, episode any if rec.Season != nil { season = *rec.Season @@ -434,12 +453,12 @@ func (r *PeernetRecordRepository) insert(ctx context.Context, rec PeernetRecord, if _, err := tx.ExecContext(ctx, `INSERT INTO peernet_records (network_id, seq, kind, content_hash, release_id, file_id, publisher_node_id, created_at, expires_at, received_at, source, relayer_node_id, healthy, backbones, checked_at, segments_checked, total_segments, - imdb_id, season, episode, title, total_size, payload) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, + imdb_id, season, episode, title, total_size, payload, release_scheme, bundle_id) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, rec.NetworkID, seq, rec.Kind, rec.ContentHash, rec.ReleaseID, rec.FileID, rec.PublisherNodeID, rec.CreatedAt.Unix(), rec.ExpiresAt.Unix(), now.Unix(), rec.Source, rec.RelayerNodeID, healthy, canonicalBackbones(rec.Backbones), unixOrZero(checked), rec.SegmentsChecked, rec.TotalSegments, - rec.IMDbID, season, episode, rec.Title, rec.TotalSize, rec.Payload); err != nil { + rec.IMDbID, season, episode, rec.Title, rec.TotalSize, rec.Payload, scheme, rec.BundleID); err != nil { return PeernetInsertResult{}, fmt.Errorf("insert peernet record: %w", err) } if err := addRowCounts(ctx, tx, rec.NetworkID, recordRef{publisher: rec.PublisherNodeID, source: rec.Source, relayer: rec.RelayerNodeID}, 1); err != nil { @@ -448,6 +467,9 @@ func (r *PeernetRecordRepository) insert(ctx context.Context, rec PeernetRecord, if err := tx.Commit(); err != nil { return PeernetInsertResult{}, fmt.Errorf("insert peernet record: commit: %w", err) } + if rec.BundleID != "" { + bundleOfferGeneration.Add(1) + } if superseded { return PeernetInsertResult{Outcome: PeernetRecordSuperseded, Seq: seq}, nil } @@ -814,3 +836,25 @@ func (r *PeernetRecordRepository) NetworkIDs(ctx context.Context) ([]string, err } return out, rows.Err() } + +// bundleOfferGeneration counts the records stored with a Bundle ID in this +// process. A cached "this node holds a bundle offer" answer is stale exactly +// when the number changed. +var bundleOfferGeneration atomic.Uint64 + +// BundleOfferGeneration returns the number of bundle-announcing records stored +// by this process so far (see AnyBundleOffer). +func BundleOfferGeneration() uint64 { return bundleOfferGeneration.Load() } + +// AnyBundleOffer reports whether any stored record announces a bundle. +func (r *PeernetRecordRepository) AnyBundleOffer(ctx context.Context) (bool, error) { + var one int + err := r.db.QueryRowContext(ctx, `SELECT 1 FROM peernet_records WHERE bundle_id <> '' LIMIT 1`).Scan(&one) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("peernet records: any bundle offer: %w", err) + } + return true, nil +} diff --git a/internal/database/repository.go b/internal/database/repository.go index 7b003c4c6..3352e7556 100644 --- a/internal/database/repository.go +++ b/internal/database/repository.go @@ -347,16 +347,17 @@ func (r *Repository) GetQueueItem(ctx context.Context, id int64) (*ImportQueueIt query := ` SELECT id, download_id, nzb_path, relative_path, category, priority, status, created_at, updated_at, started_at, completed_at, retry_count, max_retries, error_message, batch_id, metadata, file_size, storage_path, target_path, indexer, - skip_arr_notification, skip_post_import_links + skip_arr_notification, skip_post_import_links, peer_hold FROM import_queue WHERE id = ? ` var item ImportQueueItem + var peerHold int err := r.db.QueryRowContext(ctx, query, id).Scan( &item.ID, &item.DownloadID, &item.NzbPath, &item.RelativePath, &item.Category, &item.Priority, &item.Status, &item.CreatedAt, &item.UpdatedAt, &item.StartedAt, &item.CompletedAt, &item.RetryCount, &item.MaxRetries, &item.ErrorMessage, &item.BatchID, &item.Metadata, &item.FileSize, &item.StoragePath, &item.TargetPath, &item.Indexer, - &item.SkipArrNotification, &item.SkipPostImportLinks, + &item.SkipArrNotification, &item.SkipPostImportLinks, &peerHold, ) if err != nil { @@ -365,6 +366,7 @@ func (r *Repository) GetQueueItem(ctx context.Context, id int64) (*ImportQueueIt } return nil, fmt.Errorf("failed to get queue item: %w", err) } + item.PeerHold = peerHold != 0 return &item, nil } @@ -374,16 +376,17 @@ func (r *Repository) GetQueueItemByDownloadID(ctx context.Context, downloadID st query := ` SELECT id, download_id, nzb_path, relative_path, category, priority, status, created_at, updated_at, started_at, completed_at, retry_count, max_retries, error_message, batch_id, metadata, file_size, storage_path, target_path, indexer, - skip_arr_notification, skip_post_import_links + skip_arr_notification, skip_post_import_links, peer_hold FROM import_queue WHERE download_id = ? ` var item ImportQueueItem + var peerHold int err := r.db.QueryRowContext(ctx, query, downloadID).Scan( &item.ID, &item.DownloadID, &item.NzbPath, &item.RelativePath, &item.Category, &item.Priority, &item.Status, &item.CreatedAt, &item.UpdatedAt, &item.StartedAt, &item.CompletedAt, &item.RetryCount, &item.MaxRetries, &item.ErrorMessage, &item.BatchID, &item.Metadata, &item.FileSize, &item.StoragePath, &item.TargetPath, &item.Indexer, - &item.SkipArrNotification, &item.SkipPostImportLinks, + &item.SkipArrNotification, &item.SkipPostImportLinks, &peerHold, ) if err != nil { @@ -392,6 +395,7 @@ func (r *Repository) GetQueueItemByDownloadID(ctx context.Context, downloadID st } return nil, fmt.Errorf("failed to get queue item by download_id: %w", err) } + item.PeerHold = peerHold != 0 return &item, nil } @@ -881,15 +885,37 @@ func (r *Repository) ListQueueItems(ctx context.Context, status *QueueStatus, se // ListActiveQueueItems retrieves pending and processing queue items func (r *Repository) ListActiveQueueItems(ctx context.Context, search string, category string, limit, offset int, sortBy, sortOrder string) ([]*ImportQueueItem, error) { + return r.listActiveQueueItems(ctx, false, search, category, limit, offset, sortBy, sortOrder) +} + +// ListActiveQueueItemsWithHeld is ListActiveQueueItems plus the completed items +// a peer bundle import still holds back (peer_hold set): the SABnzbd-compatible +// queue shows them as work in progress. +func (r *Repository) ListActiveQueueItemsWithHeld(ctx context.Context, search string, category string, limit, offset int, sortBy, sortOrder string) ([]*ImportQueueItem, error) { + return r.listActiveQueueItems(ctx, true, search, category, limit, offset, sortBy, sortOrder) +} + +const activeQueueStatuses = "(status = 'pending' OR status = 'processing' OR status = 'paused')" + +// activeQueueCondition is the status condition of the active queue, widened by +// the held peer imports when withHeld is set. +func activeQueueCondition(withHeld bool) string { + if withHeld { + return "(" + activeQueueStatuses + " OR peer_hold <> 0)" + } + return activeQueueStatuses +} + +func (r *Repository) listActiveQueueItems(ctx context.Context, withHeld bool, search string, category string, limit, offset int, sortBy, sortOrder string) ([]*ImportQueueItem, error) { var query string var args []any baseSelect := `SELECT id, download_id, nzb_path, relative_path, category, priority, status, created_at, updated_at, started_at, completed_at, retry_count, max_retries, error_message, batch_id, metadata, file_size, storage_path, target_path, indexer, - skip_arr_notification, skip_post_import_links + skip_arr_notification, skip_post_import_links, peer_hold FROM import_queue` - conditions := []string{"(status = 'pending' OR status = 'processing' OR status = 'paused')"} + conditions := []string{activeQueueCondition(withHeld)} var conditionArgs []any if search != "" { @@ -937,15 +963,17 @@ func (r *Repository) ListActiveQueueItems(ctx context.Context, search string, ca var items []*ImportQueueItem for rows.Next() { var item ImportQueueItem + var peerHold int err := rows.Scan( &item.ID, &item.DownloadID, &item.NzbPath, &item.RelativePath, &item.Category, &item.Priority, &item.Status, &item.CreatedAt, &item.UpdatedAt, &item.StartedAt, &item.CompletedAt, &item.RetryCount, &item.MaxRetries, &item.ErrorMessage, &item.BatchID, &item.Metadata, &item.FileSize, &item.StoragePath, &item.TargetPath, &item.Indexer, - &item.SkipArrNotification, &item.SkipPostImportLinks, + &item.SkipArrNotification, &item.SkipPostImportLinks, &peerHold, ) if err != nil { return nil, fmt.Errorf("failed to scan queue item: %w", err) } + item.PeerHold = peerHold != 0 items = append(items, &item) } @@ -997,10 +1025,19 @@ func (r *Repository) CountQueueItems(ctx context.Context, status *QueueStatus, s // CountActiveQueueItems counts the total number of pending and processing queue items func (r *Repository) CountActiveQueueItems(ctx context.Context, search string, category string) (int, error) { + return r.countActiveQueueItems(ctx, false, search, category) +} + +// CountActiveQueueItemsWithHeld counts what ListActiveQueueItemsWithHeld lists. +func (r *Repository) CountActiveQueueItemsWithHeld(ctx context.Context, search string, category string) (int, error) { + return r.countActiveQueueItems(ctx, true, search, category) +} + +func (r *Repository) countActiveQueueItems(ctx context.Context, withHeld bool, search string, category string) (int, error) { var query string var args []any - baseQuery := `SELECT COUNT(*) FROM import_queue WHERE (status = 'pending' OR status = 'processing' OR status = 'paused')` + baseQuery := `SELECT COUNT(*) FROM import_queue WHERE ` + activeQueueCondition(withHeld) var conditions []string var conditionArgs []any @@ -1366,7 +1403,7 @@ func sabnzbdHistoryUnion(category string) (string, []any) { q.file_size, q.completed_at, q.category, q.storage_path, q.metadata, q.error_message, COALESCE(q.completed_at, q.updated_at, q.created_at) AS sort_time FROM import_queue q - WHERE q.status = 'completed' AND q.skip_arr_notification = FALSE + WHERE q.status = 'completed' AND q.skip_arr_notification = FALSE AND q.peer_hold = 0 AND (? = '' OR LOWER(q.category) = LOWER(?)) UNION ALL SELECT 'history', COALESCE(h.nzb_id, h.id), h.download_id, h.nzb_name, 'completed', diff --git a/internal/database/testing_test.go b/internal/database/testing_test.go index c534bae63..4b769f20a 100644 --- a/internal/database/testing_test.go +++ b/internal/database/testing_test.go @@ -34,6 +34,7 @@ func setupQueueSchema(t *testing.T, db *sql.DB) { target_path TEXT DEFAULT NULL, skip_arr_notification BOOLEAN NOT NULL DEFAULT FALSE, skip_post_import_links BOOLEAN NOT NULL DEFAULT FALSE, + peer_hold INTEGER NOT NULL DEFAULT 0, indexer TEXT DEFAULT NULL, UNIQUE(nzb_path) ); diff --git a/internal/health/checker.go b/internal/health/checker.go index dc5f703f2..e99eba48c 100644 --- a/internal/health/checker.go +++ b/internal/health/checker.go @@ -93,6 +93,17 @@ type CheckOptions struct { // knows about each due file, keyed by path. Peer advice is considered only // for files listed here; nil (every other caller) means never. PeerFiles map[string]PeerFileInfo + // FullCheckFiles lists files that owe a full check of every segment (peer + // bundle imports). They are sampled at 100 percent whatever the configured + // sample, and never get peer advice. + FullCheckFiles map[string]struct{} +} + +// owesFullCheck reports whether the file is a peer-bundle import that must be +// checked in full. +func owesFullCheck(o CheckOptions, filePath string) bool { + _, ok := o.FullCheckFiles[filePath] + return ok } // PatchIndex reports articles restored locally by PAR2 repair. @@ -295,7 +306,7 @@ func (hc *HealthChecker) prepareCheck(ctx context.Context, filePath string, opts } // Override sample percentage if forced full check is requested - if len(opts) > 0 && opts[0].ForceFullCheck { + if len(opts) > 0 && (opts[0].ForceFullCheck || owesFullCheck(opts[0], filePath)) { samplePercentage = 100 slog.InfoContext(ctx, "Forcing full health check (100% sampling)", "file_path", filePath) } diff --git a/internal/health/peer_advice.go b/internal/health/peer_advice.go index b41bc25b9..e4000f40d 100644 --- a/internal/health/peer_advice.go +++ b/internal/health/peer_advice.go @@ -146,6 +146,10 @@ func (hc *HealthChecker) peerEligible(ctx context.Context, prep *preparedCheck, return PeerFileInfo{}, false } o := opts[0] + if owesFullCheck(o, prep.filePath) { + // A peer-bundle import owes a full check: advice never applies. + return PeerFileInfo{}, false + } info, ok := o.PeerFiles[prep.filePath] switch { case !ok, info.Status != database.HealthStatusHealthy: diff --git a/internal/health/peer_bundle.go b/internal/health/peer_bundle.go new file mode 100644 index 000000000..fdbfe5616 --- /dev/null +++ b/internal/health/peer_bundle.go @@ -0,0 +1,106 @@ +package health + +import ( + "context" + "log/slog" + + "github.com/kipsilabs/altmount/internal/database" +) + +// PeerBundleGate is what the health worker needs to know about imports that +// came from a peer bundle. Such a file is playable at once but owes a full +// check of every segment: until it passes, the import is not announced to the +// ARRs and not published to the peer network, and peer advice never applies to +// it. The interface is defined here so this package never imports the peer +// network; the cmd wiring installs an adapter and removes it while the feature +// is off, so with no gate nothing changes and nothing extra runs (no query). +type PeerBundleGate interface { + // PendingFullCheck returns the subset of paths whose full check is pending. + // It must fail closed (report pending on an error). + PendingFullCheck(ctx context.Context, paths []string) map[string]struct{} + // IsPendingPeerImport reports whether the file is a peer import whose full + // check is pending. Unlike PendingFullCheck it fails open (false on an + // error): it decides to stop repair attempts for a file, which a database + // hiccup must not do to an ordinary one. + IsPendingPeerImport(ctx context.Context, filePath string) bool + // FullCheckPassed is called after a pending file's full check came back + // healthy and its status was written. + FullCheckPassed(ctx context.Context, filePath string) + // FullCheckFailed is called when a pending file's full check failed + // definitively (corrupted, retries exhausted). The gate fails the whole + // import (the queue item, the ARR-visible history entry) and removes its + // files; the worker never tries an ARR repair for such a file. It is + // idempotent. + FullCheckFailed(ctx context.Context, filePath string) +} + +type peerBundleGateHolder struct{ g PeerBundleGate } + +// SetPeerBundleGate installs (or, with nil, removes) the gate. Safe at any time. +func (hw *HealthWorker) SetPeerBundleGate(g PeerBundleGate) { + if g == nil { + hw.peerBundleGate.Store(nil) + return + } + hw.peerBundleGate.Store(&peerBundleGateHolder{g: g}) +} + +// pendingPeerBundles returns which paths owe a full check; nil with no gate. +func (hw *HealthWorker) pendingPeerBundles(ctx context.Context, paths []string) map[string]struct{} { + h := hw.peerBundleGate.Load() + if h == nil || len(paths) == 0 { + return nil + } + defer func() { + if r := recover(); r != nil { + slog.ErrorContext(ctx, "Peer bundle gate panicked", "panic", r) + } + }() + return h.g.PendingFullCheck(ctx, paths) +} + +// notifyPeerBundleResults tells the gate how the full checks of pending peer +// files ended. Only results whose write landed count; a retry or inconclusive +// result is no verdict. +func (hw *HealthWorker) notifyPeerBundleResults(ctx context.Context, pending map[string]struct{}, updates []database.HealthStatusUpdate, applied []bool, events map[string]HealthEvent) { + h := hw.peerBundleGate.Load() + if h == nil || len(pending) == 0 { + return + } + defer func() { + if r := recover(); r != nil { + slog.ErrorContext(ctx, "Peer bundle gate panicked", "panic", r) + } + }() + for i, u := range updates { + if _, ok := pending[u.FilePath]; !ok || i >= len(applied) || !applied[i] { + continue + } + ev, ok := events[u.FilePath] + if !ok { + continue + } + switch { + case u.Type == database.UpdateTypeHealthy && ev.Type == EventTypeFileHealthy: + h.g.FullCheckPassed(ctx, u.FilePath) + case ev.Type == EventTypeFileCorrupted && u.Type != database.UpdateTypeRetry && + u.Type != database.UpdateTypeInconclusive && u.Type != database.UpdateTypeDeferred && !u.Skip: + h.g.FullCheckFailed(ctx, u.FilePath) + } + } +} + +// peerImportFailsFor reports whether a corrupted verdict for the file ends a +// pending peer import (and so is handled by the gate, never by an ARR repair). +func (hw *HealthWorker) peerImportFailsFor(ctx context.Context, filePath string) bool { + h := hw.peerBundleGate.Load() + if h == nil { + return false + } + defer func() { + if r := recover(); r != nil { + slog.ErrorContext(ctx, "Peer bundle gate panicked", "panic", r) + } + }() + return h.g.IsPendingPeerImport(ctx, filePath) +} diff --git a/internal/health/peer_bundle_test.go b/internal/health/peer_bundle_test.go new file mode 100644 index 000000000..991f65614 --- /dev/null +++ b/internal/health/peer_bundle_test.go @@ -0,0 +1,211 @@ +package health + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeGate struct { + mu sync.Mutex + pending map[string]struct{} + queried int + passed []string + failed []string +} + +func (g *fakeGate) PendingFullCheck(_ context.Context, paths []string) map[string]struct{} { + g.mu.Lock() + defer g.mu.Unlock() + g.queried++ + out := map[string]struct{}{} + for _, p := range paths { + if _, ok := g.pending[p]; ok { + out[p] = struct{}{} + } + } + return out +} +func (g *fakeGate) IsPendingPeerImport(_ context.Context, p string) bool { + g.mu.Lock() + defer g.mu.Unlock() + _, ok := g.pending[p] + return ok +} +func (g *fakeGate) FullCheckPassed(_ context.Context, p string) { + g.mu.Lock() + g.passed = append(g.passed, p) + g.mu.Unlock() +} +func (g *fakeGate) FullCheckFailed(_ context.Context, p string) { + g.mu.Lock() + g.failed = append(g.failed, p) + g.mu.Unlock() +} + +func TestPeerBundle_FullCheckFilesAreSampledInFullAndNeverGetAdvice(t *testing.T) { + env := sampledEnv(t) + adv := &fakeAdviser{advice: PeerAdvice{Kind: PeerAdviceReduceSample}} + env.checker.SetPeerAdviser(adv) + opts := autoOpts(env, database.HealthStatusHealthy) + opts.FullCheckFiles = map[string]struct{}{env.filePath: {}} + + var ev HealthEvent + n := statCount(env, func() { ev = env.checker.CheckFile(context.Background(), env.filePath, opts) }) + require.Equal(t, EventTypeFileHealthy, ev.Type) + assert.Equal(t, bigFileSegments, n, "every segment is checked") + assert.True(t, ev.Stats.FullCheck) + assert.Zero(t, adv.consulted(), "peer advice never applies to a pending peer file") + assert.False(t, ev.Stats.ReducedByPeerAdvice) +} + +func TestPeerBundle_AdviceGuardHoldsEvenWhenTheSampleIsNotForced(t *testing.T) { + env := sampledEnv(t) + adv := &fakeAdviser{advice: PeerAdvice{Kind: PeerAdviceSkipOnce, SkipUntil: time.Now().Add(5 * 24 * time.Hour)}} + env.checker.SetPeerAdviser(adv) + prep := &preparedCheck{filePath: env.filePath, realSegments: bigFileSegments} + opts := autoOpts(env, database.HealthStatusHealthy) + opts.FullCheckFiles = map[string]struct{}{env.filePath: {}} + _, ok := env.checker.peerEligible(context.Background(), prep, nil, []CheckOptions{opts}, env.cfg) + assert.False(t, ok, "a pending peer file is not eligible") + opts.FullCheckFiles = nil + _, ok = env.checker.peerEligible(context.Background(), prep, nil, []CheckOptions{opts}, env.cfg) + assert.True(t, ok, "control: the same file is eligible without the guard") +} + +func TestPeerBundle_CyclePassNotifiesGateOnce(t *testing.T) { + ctx := context.Background() + env, h := cycleEnv(t) + adv := &fakeAdviser{advice: PeerAdvice{Kind: PeerAdviceSkipOnce, SkipUntil: time.Now().Add(5 * 24 * time.Hour)}} + env.hw.SetPeerAdviser(adv) + gate := &fakeGate{pending: map[string]struct{}{healthPath(h): {}}} + env.hw.SetPeerBundleGate(gate) + + n := statCount(h, func() { require.NoError(t, env.hw.runHealthCheckCycle(ctx)) }) + assert.Equal(t, bigFileSegments, n, "full check, not deferred") + assert.Zero(t, adv.consulted()) + assert.Equal(t, []string{healthPath(h)}, gate.passed) + assert.Empty(t, gate.failed) +} + +func TestPeerBundle_CycleFailureNotifiesGate(t *testing.T) { + ctx := context.Background() + env, h := cycleEnv(t) + h.markSegmentMissing(bigFileSegments / 2) // only a full check can see this + gate := &fakeGate{pending: map[string]struct{}{healthPath(h): {}}} + env.hw.SetPeerBundleGate(gate) + _, err := env.db.ExecContext(ctx, `UPDATE file_health SET retry_count = ?`, h.cfg.GetMaxRetries()-1) + require.NoError(t, err) + + require.NoError(t, env.hw.runHealthCheckCycle(ctx)) + assert.Empty(t, gate.passed) + assert.Equal(t, []string{healthPath(h)}, gate.failed) +} + +func TestPeerBundle_NoGateMeansNoLookup(t *testing.T) { + ctx := context.Background() + env, h := cycleEnv(t) + assert.Nil(t, env.hw.pendingPeerBundles(ctx, []string{healthPath(h)})) + gate := &fakeGate{} + env.hw.SetPeerBundleGate(gate) + env.hw.SetPeerBundleGate(nil) + require.NoError(t, env.hw.runHealthCheckCycle(ctx)) + assert.Zero(t, gate.queried, "removed gate is never asked") +} + +func TestPeerBundle_OnlyDefinitiveVerdictsReachTheGate(t *testing.T) { + const p = "m/a.mkv" + for name, tc := range map[string]struct { + update database.HealthStatusUpdate + event EventType + applied bool + passed int + failed int + }{ + "retry is no verdict": {database.HealthStatusUpdate{FilePath: p, Type: database.UpdateTypeRetry}, EventTypeFileCorrupted, true, 0, 0}, + "inconclusive": {database.HealthStatusUpdate{FilePath: p, Type: database.UpdateTypeInconclusive}, EventTypeCheckInconclusive, true, 0, 0}, + "check failed w/o verdict": {database.HealthStatusUpdate{FilePath: p, Type: database.UpdateTypeRetry}, EventTypeCheckFailed, true, 0, 0}, + "write lost a race": {database.HealthStatusUpdate{FilePath: p, Type: database.UpdateTypeHealthy}, EventTypeFileHealthy, false, 0, 0}, + "healthy": {database.HealthStatusUpdate{FilePath: p, Type: database.UpdateTypeHealthy}, EventTypeFileHealthy, true, 1, 0}, + "repair triggered": {database.HealthStatusUpdate{FilePath: p, Type: database.UpdateTypeRepairTrigger}, EventTypeFileCorrupted, true, 0, 1}, + "corrupted": {database.HealthStatusUpdate{FilePath: p, Type: database.UpdateTypeCorrupted}, EventTypeFileCorrupted, true, 0, 1}, + } { + t.Run(name, func(t *testing.T) { + hw := &HealthWorker{} + gate := &fakeGate{} + hw.SetPeerBundleGate(gate) + hw.notifyPeerBundleResults(context.Background(), map[string]struct{}{p: {}}, + []database.HealthStatusUpdate{tc.update}, []bool{tc.applied}, map[string]HealthEvent{p: {Type: tc.event, FilePath: p}}) + assert.Len(t, gate.passed, tc.passed) + assert.Len(t, gate.failed, tc.failed) + }) + } +} + +// orderObserver records whether the gate had already been told the full check +// passed when the healthy notice (which the publisher acts on) arrived. +type orderObserver struct { + gate *fakeGate + passedFirst []bool +} + +func (o *orderObserver) FileHealthy(n HealthyNotice) { + o.gate.mu.Lock() + defer o.gate.mu.Unlock() + o.passedFirst = append(o.passedFirst, len(o.gate.passed) > 0) +} + +func TestPeerBundle_GateIsToldBeforeThePublisherSeesTheHealthyNotice(t *testing.T) { + ctx := context.Background() + env, h := cycleEnv(t) + gate := &fakeGate{pending: map[string]struct{}{healthPath(h): {}}} + obs := &orderObserver{gate: gate} + env.hw.SetPeerBundleGate(gate) + env.hw.SetHealthyObserver(obs) + require.NoError(t, env.hw.runHealthCheckCycle(ctx)) + require.Equal(t, []bool{true}, obs.passedFirst) +} + +// A pending peer import whose full check fails for good must not reach the +// ARR repair path (an ARR that grabbed the item learns of the failure through +// its history; a direct import has no ARR): the gate is told, once, and the +// ARR is never called. An ordinary file in the same state still is repaired. +func TestPeerBundle_FailedFullCheckNeverTriggersAnARRRepair(t *testing.T) { + for name, pending := range map[string]bool{"pending peer import": true, "ordinary file (control)": false} { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + env := newRepairTestEnv(t, t.TempDir(), nil) + filePath := "series/show.s01e01.mkv" + libraryPath := filepath.Join(t.TempDir(), "show.s01e01.mkv") + require.NoError(t, os.WriteFile(libraryPath, []byte("x"), 0o644)) + require.NoError(t, env.metadataService.WriteFileMetadata(filePath, validSegmentMeta(env.metadataService, 1024))) + insertFileHealth(t, env.db, filePath, libraryPath, 2, 3) + + gate := &fakeGate{pending: map[string]struct{}{}} + if pending { + gate.pending[filePath] = struct{}{} + } + env.hw.SetPeerBundleGate(gate) + + require.NoError(t, env.hw.runHealthCheckCycle(ctx)) + + env.mockARRs.mu.Lock() + calls := len(env.mockARRs.calls) + env.mockARRs.mu.Unlock() + if pending { + assert.Zero(t, calls, "no ARR repair for a failed peer import") + assert.Equal(t, []string{filePath}, gate.failed, "the gate fails the import exactly once") + return + } + assert.Equal(t, 1, calls, "an ordinary file is repaired as before") + assert.Empty(t, gate.failed) + }) + } +} diff --git a/internal/health/worker.go b/internal/health/worker.go index 4317e073d..9bed82308 100644 --- a/internal/health/worker.go +++ b/internal/health/worker.go @@ -90,6 +90,9 @@ type HealthWorker struct { // healthyObserver is notified of fresh healthy results (see observer.go). // Nil by default: no behaviour change and no overhead. healthyObserver atomic.Pointer[observerHolder] + // peerBundleGate holds back peer-bundle imports until their full check + // passed (see peer_bundle.go). Nil: feature off, nothing extra runs. + peerBundleGate atomic.Pointer[peerBundleGateHolder] // Worker state status WorkerStatus @@ -638,6 +641,23 @@ func (hw *HealthWorker) prepareUpdateForResult(ctx context.Context, fh *database // When delete-on-corruption is enabled, confirmed corruption is handled by removing // the file entirely instead of ever entering the repair flow below (this covers both // freshly-exhausted checks and files already sitting in repair_triggered). + // A pending peer import that came back corrupted for good (retries used up, + // or delete-on-corruption) is not repaired through an ARR: the corrupted + // verdict is written and the peer-bundle gate is told once the write landed + // (notifyPeerBundleResults), which fails the whole import: the queue item + // turns into a failed one (an ARR that grabbed it sees that in its history + // and searches again) and its files, this one included, are removed. The + // same item is therefore never acted on twice. + if (hw.configGetter().GetHealthDeleteOnCorruption() || fh.RetryCount >= hw.configGetter().GetMaxRetries()-1) && + hw.peerImportFailsFor(ctx, fh.FilePath) { + update.Type = database.UpdateTypeCorrupted + update.Status = database.HealthStatusCorrupted + return update, func() error { + slog.WarnContext(ctx, "Peer import failed its full check; no ARR repair is triggered", "file_path", fh.FilePath) + return nil + } + } + if hw.configGetter().GetHealthDeleteOnCorruption() { return hw.deleteCorruptedFile(ctx, fh) } @@ -960,6 +980,11 @@ func (hw *HealthWorker) performDirectCheck(ctx context.Context, filePath string, } opts := CheckOptions{CurrentStatus: currentStatus, VerifyContentOverride: verifyContentOverride, OnProgress: hw.progressReporter(map[string]*activeCheck{filePath: entry})} + // A peer-bundle import owes a full check, whoever asked for this one. + peerPending := hw.pendingPeerBundles(ctx, []string{filePath}) + if len(peerPending) > 0 { + opts.FullCheckFiles = peerPending + } // Delegate to HealthChecker event := hw.healthChecker.CheckFile(checkCtx, filePath, opts) @@ -999,6 +1024,9 @@ func (hw *HealthWorker) performDirectCheck(ctx context.Context, filePath string, if err != nil { return fmt.Errorf("failed to update health status: %w", err) } + // The gate first: the publisher acts on the healthy notice and must + // find a passed full check already recorded. + hw.notifyPeerBundleResults(ctx, peerPending, []database.HealthStatusUpdate{*updatePtr}, applied, map[string]HealthEvent{filePath: event}) hw.notifyHealthyApplied([]database.HealthStatusUpdate{*updatePtr}, applied, map[string]HealthEvent{filePath: event}) hw.enqueueDegradedRepair(ctx, *updatePtr) hw.broadcastHealthChanged() @@ -1110,6 +1138,7 @@ func (hw *HealthWorker) runHealthCheckCycle(ctx context.Context) error { // healthyEvents holds, only while an observer is installed, the healthy // results to announce once their writes have landed. healthyEvents := map[string]HealthEvent{} + peerEvents := map[string]HealthEvent{} var resultsMu sync.Mutex // The regular-check writes are based on the record being 'checking' (set just above); @@ -1137,6 +1166,11 @@ func (hw *HealthWorker) runHealthCheckCycle(ctx context.Context) error { statuses[i] = fh.Status } checkOpts := CheckOptions{OnProgress: reportProgress} + // Peer-bundle imports owe a full check (nil with no gate: no lookup). + peerPending := hw.pendingPeerBundles(ctx, paths) + if len(peerPending) > 0 { + checkOpts.FullCheckFiles = peerPending + } if hw.healthChecker.peerAdviserOn() { checkOpts.PeerFiles = peerFilesFor(unhealthyFiles) } @@ -1169,6 +1203,9 @@ func (hw *HealthWorker) runHealthCheckCycle(ctx context.Context) error { if hw.healthyObserver.Load() != nil && event.Type == EventTypeFileHealthy { healthyEvents[fh.FilePath] = event } + if _, ok := peerPending[fh.FilePath]; ok { + peerEvents[fh.FilePath] = event + } resultsMu.Unlock() // Notify VFS @@ -1271,6 +1308,8 @@ func (hw *HealthWorker) runHealthCheckCycle(ctx context.Context) error { for _, result := range results { hw.enqueueDegradedRepair(ctx, result) } + // The gate first (see the direct check path). + hw.notifyPeerBundleResults(ctx, peerPending, results, applied, peerEvents) hw.notifyHealthyApplied(results, applied, healthyEvents) } hw.broadcastHealthChanged() diff --git a/internal/importer/ensure_persistent_nzb_test.go b/internal/importer/ensure_persistent_nzb_test.go index accaeb821..51565e739 100644 --- a/internal/importer/ensure_persistent_nzb_test.go +++ b/internal/importer/ensure_persistent_nzb_test.go @@ -55,6 +55,7 @@ func newMinimalServiceForPersistTest(t *testing.T) *Service { target_path TEXT DEFAULT NULL, skip_arr_notification BOOLEAN NOT NULL DEFAULT FALSE, skip_post_import_links BOOLEAN NOT NULL DEFAULT FALSE, + peer_hold INTEGER NOT NULL DEFAULT 0, indexer TEXT DEFAULT NULL, UNIQUE(nzb_path) ); diff --git a/internal/importer/filesystem/utils.go b/internal/importer/filesystem/utils.go index 400f510ef..dda939d08 100644 --- a/internal/importer/filesystem/utils.go +++ b/internal/importer/filesystem/utils.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strings" "sync" + "unicode" + "unicode/utf8" "github.com/kipsilabs/altmount/internal/importer/parser" "github.com/kipsilabs/altmount/internal/importer/utils/nzbtrim" @@ -120,6 +122,18 @@ func EnsureDirectoryExists(virtualDir string, metadataService *metadata.Metadata // CreateNzbFolder creates a folder named after the NZB file func CreateNzbFolder(virtualDir, nzbFilename string, metadataService *metadata.MetadataService) (string, error) { + nzbVirtualDir := NzbFolderPath(virtualDir, nzbFilename) + + if err := EnsureDirectoryExists(nzbVirtualDir, metadataService); err != nil { + return "", err + } + + return nzbVirtualDir, nil +} + +// NzbFolderPath is the virtual folder CreateNzbFolder would create, without +// creating anything. +func NzbFolderPath(virtualDir, nzbFilename string) string { nzbBaseName := nzbtrim.TrimNzbExtension(nzbFilename) // Now, also strip the media file extension if it exists // Common media extensions: .mkv, .mp4, .avi, .flv, .wmv, .mov, .webm @@ -134,13 +148,7 @@ func CreateNzbFolder(virtualDir, nzbFilename string, metadataService *metadata.M } nzbVirtualDir := filepath.Join(virtualDir, nzbBaseName) - nzbVirtualDir = strings.ReplaceAll(nzbVirtualDir, string(filepath.Separator), "/") - - if err := EnsureDirectoryExists(nzbVirtualDir, metadataService); err != nil { - return "", err - } - - return nzbVirtualDir, nil + return strings.ReplaceAll(nzbVirtualDir, string(filepath.Separator), "/") } // CreateDirectoriesForFiles analyzes files and creates their parent directories @@ -312,3 +320,58 @@ func (r *PathReserver) Release(path string) { delete(r.claimed, path) r.mu.Unlock() } + +// ReservedFolderNames are folder names the application uses for its own +// bookkeeping inside the library; no peer-supplied title may become one. +var ReservedFolderNames = []string{".nzbs", "corrupted_metadata", "patches", "_migrated", ".peernet-staging"} + +// maxSafeFolderNameLen caps a peer-derived folder name in bytes. +const maxSafeFolderNameLen = 200 + +// SafeFolderName turns an untrusted title into one plain folder name. The media +// extension is stripped first (as NzbFolderPath does later), then control and +// path characters are replaced, leading and trailing dots and spaces removed +// and the length capped. A name that ends up empty, starts with a dot, or +// equals (ignoring case) a reserved internal name or one of extraReserved (for +// example the category folders) is refused. Unlike NzbFolderPath it can never +// yield "." or "..". +func SafeFolderName(title string, extraReserved ...string) (string, bool) { + name := strings.ToValidUTF8(title, "") + if i := strings.LastIndexAny(name, `/\`); i >= 0 { + name = name[i+1:] + } + if nzbtrim.HasNzbExtension(name) { + name = nzbtrim.TrimNzbExtension(name) + } + lower := strings.ToLower(name) + for _, ext := range []string{".mkv", ".mp4", ".avi", ".flv", ".wmv", ".mov", ".webm", ".ts", ".iso"} { + if strings.HasSuffix(lower, ext) { + name = name[:len(name)-len(ext)] + break + } + } + name = strings.Map(func(r rune) rune { + if unicode.IsControl(r) || strings.ContainsRune(`<>:"|?*`, r) { + return ' ' + } + return r + }, name) + name = strings.Join(strings.Fields(name), " ") + if strings.HasPrefix(name, ".") { + return "", false + } + for len(name) > maxSafeFolderNameLen { + _, size := utf8.DecodeLastRuneInString(name) + name = name[:len(name)-size] + } + name = strings.TrimRight(name, ". ") + if name == "" || strings.HasPrefix(name, ".") { + return "", false + } + for _, r := range append(append([]string{}, ReservedFolderNames...), extraReserved...) { + if r != "" && strings.EqualFold(name, r) { + return "", false + } + } + return name, true +} diff --git a/internal/importer/handle_failure_test.go b/internal/importer/handle_failure_test.go index 619ee16f3..f0c43f38d 100644 --- a/internal/importer/handle_failure_test.go +++ b/internal/importer/handle_failure_test.go @@ -61,6 +61,7 @@ func newMoveToFailedTestService(t *testing.T) *Service { target_path TEXT DEFAULT NULL, skip_arr_notification BOOLEAN NOT NULL DEFAULT FALSE, skip_post_import_links BOOLEAN NOT NULL DEFAULT FALSE, + peer_hold INTEGER NOT NULL DEFAULT 0, indexer TEXT DEFAULT NULL, UNIQUE(nzb_path) ); diff --git a/internal/importer/nzb_password_origin.go b/internal/importer/nzb_password_origin.go new file mode 100644 index 000000000..5749e05cf --- /dev/null +++ b/internal/importer/nzb_password_origin.go @@ -0,0 +1,104 @@ +package importer + +import ( + "context" + "encoding/json" + "strings" + + "github.com/kipsilabs/altmount/internal/importer/parser" +) + +// passwordInjectedKey marks, in a queue item's metadata JSON, that the API +// embedded a user-supplied password into the NZB it enqueued (the "&p=" of a +// link or the password field of a form). From then on that password is +// indistinguishable from one the NZB came with, so the mark is how the +// importer tells them apart. The password itself is never stored here. +const passwordInjectedKey = "_altmount_pw_injected" + +// MarkPasswordInjected returns queue metadata JSON that records an injected +// password, keeping every key the metadata already had. Unparsable existing +// metadata is replaced (it carried no readable key anyway). +func MarkPasswordInjected(metadata *string) *string { + fields := map[string]json.RawMessage{} + if metadata != nil && strings.TrimSpace(*metadata) != "" { + if err := json.Unmarshal([]byte(*metadata), &fields); err != nil || fields == nil { + fields = map[string]json.RawMessage{} + } + } + fields[passwordInjectedKey] = json.RawMessage("true") + out, err := json.Marshal(fields) + if err != nil { + return metadata + } + s := string(out) + return &s +} + +// passwordInjected reports whether the queue metadata carries the mark. +func passwordInjected(metadata *string) bool { + if metadata == nil || *metadata == "" { + return false + } + var fields map[string]json.RawMessage + if err := json.Unmarshal([]byte(*metadata), &fields); err != nil { + return false + } + raw, ok := fields[passwordInjectedKey] + return ok && string(raw) == "true" +} + +// NZBPasswordRecorder is told which written files were imported with a +// password that came with the NZB's own content (a meta password present in +// the NZB as received). It is installed only while the peer network is on and +// never receives the password. +type NZBPasswordRecorder interface { + RecordNZBPasswordFiles(ctx context.Context, virtualPaths []string) +} + +type nzbPasswordHolder struct{ r NZBPasswordRecorder } + +// SetNZBPasswordRecorder installs (or, with nil, removes) the recorder. +func (proc *Processor) SetNZBPasswordRecorder(r NZBPasswordRecorder) { + if r == nil { + proc.pwRecorder.Store(nil) + return + } + proc.pwRecorder.Store(&nzbPasswordHolder{r: r}) +} + +// SetNZBPasswordRecorder installs (or, with nil, removes) the recorder on the +// service's processor. +func (s *Service) SetNZBPasswordRecorder(r NZBPasswordRecorder) { + if s.processor != nil { + s.processor.SetNZBPasswordRecorder(r) + } +} + +// recordNZBPassword tells the recorder about the written files when the +// password in effect is a meta password of the NZB as received: not one of the +// name or the subjects, not one the API injected, and not one of an NZB found +// inside another release's archive. +func (proc *Processor) recordNZBPassword(ctx context.Context, parsed *parser.ParsedNzb, metadata *string, written []string) { + holder := proc.pwRecorder.Load() + if holder == nil || parsed == nil || parsed.PasswordOrigin() != parser.PasswordOriginMeta || passwordInjected(metadata) { + return + } + if chain, err := readNestedNzbChain(metadata); err != nil || chain.Depth > 0 { + return + } + paths := make([]string, 0, len(written)) + for _, p := range written { + if !strings.HasPrefix(p, "DIR:") { + paths = append(paths, p) + } + } + if len(paths) == 0 { + return + } + defer func() { + if r := recover(); r != nil { + proc.log.ErrorContext(ctx, "NZB password recorder panicked", "panic", r) + } + }() + holder.r.RecordNZBPasswordFiles(ctx, paths) +} diff --git a/internal/importer/nzb_password_origin_test.go b/internal/importer/nzb_password_origin_test.go new file mode 100644 index 000000000..4a3373685 --- /dev/null +++ b/internal/importer/nzb_password_origin_test.go @@ -0,0 +1,99 @@ +package importer + +import ( + "context" + "encoding/json" + "path/filepath" + "sync" + "testing" + + "github.com/kipsilabs/altmount/internal/testsupport/nzbbuild" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakePwRecorder struct { + mu sync.Mutex + calls [][]string +} + +func (f *fakePwRecorder) RecordNZBPasswordFiles(_ context.Context, paths []string) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, paths) +} + +// pwNZB writes a one-file NZB named name with the given meta password and +// subject suffix. +func pwNZB(e *batteryEnv, name, metaPW, subjectSuffix string) string { + content := make([]byte, 4096) + for i := range content { + content[i] = byte(i % 251) + } + segs := e.registerContent("pw", content, 1024, 1.0, nil) + n := nzbbuild.Build(nzbbuild.File{Subject: "movie.mkv" + subjectSuffix, Segments: segs}) + if metaPW != "" { + n.Meta = map[string]string{"password": metaPW} + } + return nzbbuild.WriteTemp(e.t, n, name) +} + +func runWithRecorder(t *testing.T, name, metaPW, subjectSuffix string, metadata *string) *fakePwRecorder { + t.Helper() + e := newBatteryEnv(t) + rec := &fakePwRecorder{} + e.proc.SetNZBPasswordRecorder(rec) + p := pwNZB(e, name, metaPW, subjectSuffix) + _, _, err := e.proc.ProcessNzbFile(context.Background(), p, filepath.Dir(p), 7, nil, nil, nil, nil, metadata, nil) + require.NoError(t, err) + return rec +} + +func str(s string) *string { return &s } + +func TestNZBPasswordOrigin_MetaPasswordOfTheReceivedNZBIsRecorded(t *testing.T) { + rec := runWithRecorder(t, "Some.Release", "pw-from-the-nzb", "", nil) + require.Len(t, rec.calls, 1) + assert.Equal(t, []string{"/Some.Release.mkv"}, rec.calls[0]) +} + +func TestNZBPasswordOrigin_OtherOriginsAreNotRecorded(t *testing.T) { + injected := MarkPasswordInjected(nil) + nested := str(`{"_altmount_nzb_chain":{"depth":1,"seen":[]}}`) + for name, tc := range map[string]struct { + file, metaPW, subject string + metadata *string + }{ + "password spelled in the NZB name": {file: "Some.Release{{pw-in-name}}"}, + "password spelled in a subject": {file: "Some.Release", subject: " {{pw-in-subject}}"}, + "no password at all": {file: "Some.Release"}, + "injected by the API (the &p= or form field)": {file: "Some.Release", metaPW: "typed-by-the-user", metadata: injected}, + "NZB found inside another release's archive": {file: "Some.Release", metaPW: "inner", metadata: nested}, + } { + t.Run(name, func(t *testing.T) { + assert.Empty(t, runWithRecorder(t, tc.file, tc.metaPW, tc.subject, tc.metadata).calls) + }) + } +} + +func TestNZBPasswordOrigin_NoRecorderMeansNoChange(t *testing.T) { + e := newBatteryEnv(t) + p := pwNZB(e, "Some.Release", "pw", "") + _, written, err := e.proc.ProcessNzbFile(context.Background(), p, filepath.Dir(p), 7, nil, nil, nil, nil, nil, nil) + require.NoError(t, err) + assert.Equal(t, []string{"/Some.Release.mkv"}, written) +} + +func TestMarkPasswordInjectedKeepsExistingMetadata(t *testing.T) { + got := MarkPasswordInjected(str(`{"imdb_id":"tt1","season":2}`)) + var m map[string]any + require.NoError(t, json.Unmarshal([]byte(*got), &m)) + assert.Equal(t, "tt1", m["imdb_id"]) + assert.EqualValues(t, 2, m["season"]) + assert.True(t, passwordInjected(got)) + assert.False(t, passwordInjected(str(`{"imdb_id":"tt1"}`))) + assert.False(t, passwordInjected(nil)) + assert.False(t, passwordInjected(str("not json"))) + assert.True(t, passwordInjected(MarkPasswordInjected(str("not json"))), "unreadable metadata is replaced") + assert.NotContains(t, *got, "secret", "the password itself is never in the mark") +} diff --git a/internal/importer/parser/parser.go b/internal/importer/parser/parser.go index c7fe62ba1..923c4a47c 100644 --- a/internal/importer/parser/parser.go +++ b/internal/importer/parser/parser.go @@ -212,11 +212,16 @@ func (p *Parser) ParseNzb(ctx context.Context, n *nzbparser.Nzb, nzbPath string, // Build the shared NzbStore and segment index for v3 format. // Must be built from the raw *nzbparser.Nzb BEFORE any per-file processing // (which may filter/reorder files). The index maps message-id → flat store index. - parsed.Store, parsed.SegmentIndex = metadata.BuildStore(n) + if opts.PrebuiltStore != nil && opts.PrebuiltIndex != nil { + parsed.Store, parsed.SegmentIndex = opts.PrebuiltStore, opts.PrebuiltIndex + } else { + parsed.Store, parsed.SegmentIndex = metadata.BuildStore(n) + } // Extract password from metadata, filename tags, subject lines, or raw NZB meta - if pwd := extractNzbPassword(n, nzbPath); pwd != "" { + if pwd, origin := extractNzbPasswordWithOrigin(n, nzbPath); pwd != "" { parsed.SetPassword(pwd) + parsed.passwordOrigin = origin } // Fetch first segment data for all files in parallel @@ -1864,16 +1869,34 @@ var ( passwordMetaRegex = regexp.MustCompile(`(?i)]*type\s*=\s*["']?(?:password|pass|pwd|password_hash)["']?[^>]*>([\s\S]*?)<\/meta>`) ) +// PasswordOrigin says where the password in effect for an NZB was found. +type PasswordOrigin int + +// Password origins. +const ( + PasswordOriginNone PasswordOrigin = iota // no password + PasswordOriginMeta // a in the NZB content + PasswordOriginName // spelled in the NZB file name + PasswordOriginSubject // spelled in a file subject +) + func extractNzbPassword(n *nzbparser.Nzb, nzbPath string) string { + pwd, _ := extractNzbPasswordWithOrigin(n, nzbPath) + return pwd +} + +// extractNzbPasswordWithOrigin is extractNzbPassword plus where the password +// came from. The precedence is the same. +func extractNzbPasswordWithOrigin(n *nzbparser.Nzb, nzbPath string) (string, PasswordOrigin) { if n == nil { - return "" + return "", PasswordOriginNone } // 1. Check n.Meta with case-insensitive keys if n.Meta != nil { for k, v := range n.Meta { if strings.EqualFold(k, "password") || strings.EqualFold(k, "pass") || strings.EqualFold(k, "pwd") || strings.EqualFold(k, "password_hash") { if trimmed := strings.TrimSpace(v); trimmed != "" { - return trimmed + return trimmed, PasswordOriginMeta } } } @@ -1882,24 +1905,24 @@ func extractNzbPassword(n *nzbparser.Nzb, nzbPath string) string { baseName := filepath.Base(nzbPath) if m := passwordBracketsRegex.FindStringSubmatch(baseName); len(m) > 1 { if trimmed := strings.TrimSpace(m[1]); trimmed != "" { - return trimmed + return trimmed, PasswordOriginName } } if m := passwordSlashRegex.FindStringSubmatch(baseName); len(m) > 1 { if trimmed := strings.TrimSpace(m[1]); trimmed != "" { - return trimmed + return trimmed, PasswordOriginName } } // 3. Check file subjects for {{password}} or /password:xyz/ for _, f := range n.Files { if m := passwordBracketsRegex.FindStringSubmatch(f.Subject); len(m) > 1 { if trimmed := strings.TrimSpace(m[1]); trimmed != "" { - return trimmed + return trimmed, PasswordOriginSubject } } if m := passwordSlashRegex.FindStringSubmatch(f.Subject); len(m) > 1 { if trimmed := strings.TrimSpace(m[1]); trimmed != "" { - return trimmed + return trimmed, PasswordOriginSubject } } } @@ -1912,12 +1935,12 @@ func extractNzbPassword(n *nzbparser.Nzb, nzbPath string) string { } if m := passwordMetaRegex.FindSubmatch(data[:limit]); len(m) > 1 { if trimmed := strings.TrimSpace(string(m[1])); trimmed != "" { - return trimmed + return trimmed, PasswordOriginMeta } } } } - return "" + return "", PasswordOriginNone } // propagateArchiveType sets the archive-type flag on non-PAR2 files that are diff --git a/internal/importer/parser/parser_test.go b/internal/importer/parser/parser_test.go index 819db551d..d96921cad 100644 --- a/internal/importer/parser/parser_test.go +++ b/internal/importer/parser/parser_test.go @@ -3,6 +3,8 @@ package parser import ( "context" "log/slog" + "os" + "path/filepath" "strings" "sync" "testing" @@ -15,6 +17,7 @@ import ( "github.com/kipsilabs/altmount/internal/testsupport/segments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" ) func testConfigGetter() config.ConfigGetter { @@ -365,3 +368,40 @@ func TestPropagateArchiveType_KeepsDirectMediaOutsideRar(t *testing.T) { assert.False(t, parsed.Files[4].IsRarArchive, "a sample MP4 is still direct media") assert.True(t, parsed.Files[5].IsRarArchive, "RAR magic must take priority over a misleading media extension") } + +func TestExtractNzbPasswordOrigin(t *testing.T) { + cases := []struct { + name string + n *nzbparser.Nzb + path string + pw string + origin PasswordOrigin + }{ + {"meta", &nzbparser.Nzb{Meta: map[string]string{"password": " pw1 "}}, "movie.nzb", "pw1", PasswordOriginMeta}, + {"meta, other spelling", &nzbparser.Nzb{Meta: map[string]string{"PWD": "pw2"}}, "movie.nzb", "pw2", PasswordOriginMeta}, + {"file name braces", &nzbparser.Nzb{}, "Movie{{pw3}}.nzb", "pw3", PasswordOriginName}, + {"subject slash form", &nzbparser.Nzb{Files: []nzbparser.NzbFile{{Subject: "x yEnc /password:pw4"}}}, "movie.nzb", "pw4", PasswordOriginSubject}, + {"subject", &nzbparser.Nzb{Files: []nzbparser.NzbFile{{Subject: "x yEnc {{pw5}}"}}}, "movie.nzb", "pw5", PasswordOriginSubject}, + {"none", &nzbparser.Nzb{}, "movie.nzb", "", PasswordOriginNone}, + {"meta wins over the name", &nzbparser.Nzb{Meta: map[string]string{"password": "m"}}, "Movie{{n}}.nzb", "m", PasswordOriginMeta}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + pw, origin := extractNzbPasswordWithOrigin(c.n, c.path) + assert.Equal(t, c.pw, pw) + assert.Equal(t, c.origin, origin) + }) + } + // The on-disk NZB can hold a meta password outside the parsed head. + dir := t.TempDir() + p := filepath.Join(dir, "x.nzb") + require.NoError(t, os.WriteFile(p, []byte(`ondisk`), 0o600)) + pw, origin := extractNzbPasswordWithOrigin(&nzbparser.Nzb{}, p) + assert.Equal(t, "ondisk", pw) + assert.Equal(t, PasswordOriginMeta, origin) + + parsed := &ParsedNzb{} + assert.Equal(t, PasswordOriginNone, parsed.PasswordOrigin()) + parsed.SetPassword("x") + assert.Equal(t, PasswordOriginNone, parsed.PasswordOrigin(), "an origin is only reported for a password that was extracted") +} diff --git a/internal/importer/parser/types.go b/internal/importer/parser/types.go index e4a1919f4..de3bc5145 100644 --- a/internal/importer/parser/types.go +++ b/internal/importer/parser/types.go @@ -17,6 +17,11 @@ type ParseOptions struct { // KnownMissingSegmentIDs seeds notFoundIDs so yEnc normalisation and 16KB // completion never re-issue Stat/Body calls for already-known-missing IDs. KnownMissingSegmentIDs map[string]struct{} + // PrebuiltStore and PrebuiltIndex are metadata.BuildStore of this very NZB, + // when the caller already built them and the NZB did not change since. Both + // or neither. + PrebuiltStore *metapb.NzbStore + PrebuiltIndex map[string]int64 } // NzbType represents the type of NZB content @@ -51,6 +56,8 @@ type ParsedNzb struct { // article, for files whose header was recovered from a later article. The // processor merges it into the repair queue so the hole gets patched. DegradedFiles map[string]string + // passwordOrigin says where password was found (see PasswordOrigin). + passwordOrigin PasswordOrigin } // GetPassword returns the password for this NZB @@ -58,6 +65,15 @@ func (p *ParsedNzb) GetPassword() string { return p.password } +// PasswordOrigin reports where the password in effect was found. It is +// PasswordOriginNone when there is none. +func (p *ParsedNzb) PasswordOrigin() PasswordOrigin { + if p.password == "" { + return PasswordOriginNone + } + return p.passwordOrigin +} + // SetPassword sets the password for this NZB func (p *ParsedNzb) SetPassword(password string) { p.password = password diff --git a/internal/importer/peer_bundle.go b/internal/importer/peer_bundle.go new file mode 100644 index 000000000..fac262324 --- /dev/null +++ b/internal/importer/peer_bundle.go @@ -0,0 +1,196 @@ +package importer + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "time" + + "github.com/javi11/nzbparser" + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/importer/filesystem" + "github.com/kipsilabs/altmount/internal/importer/validation" + "github.com/kipsilabs/altmount/internal/metadata" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" +) + +// PeerBundleRequest is what the peer-bundle fast path is told about the NZB +// about to be processed. It carries plain data only; this package never imports +// the peer network (internal/peernet/bridge adapts it). +type PeerBundleRequest struct { + // NZB is the segment list of the NZB the user supplied, per NZB file in NZB + // order, parsed but not yet analysed. + NZB *metapb.NzbStore + // QueueID is the queue item being processed. + QueueID int + // NzbName is the clean release name of the queue item. + NzbName string + // TargetDir is the local virtual folder the files would be written to, + // derived exactly as for a normal multi-file import. + TargetDir string + // Category is the queue item's category. + Category string + // TargetPath and SkipArr are the queue item's own notification settings. + TargetPath string + SkipArr bool +} + +// PeerBundleOutcome is a successful fast-path import. +type PeerBundleOutcome struct { + // ResultPath is the release folder; WrittenPaths are the file paths. + ResultPath string + WrittenPaths []string + TotalSize int64 +} + +// PeerBundleFastPath imports a release from a peer bundle instead of +// analysing it. It is nil by default and installed only while the peer +// network is enabled. TryBundle fails open: ok=false means "import normally", +// and it must not make an import slower than its own short deadline. +type PeerBundleFastPath interface { + // MayHaveOffers is a cheap, cached answer to "does this node hold any + // bundle offer at all". With false the import skips every peer-network step + // (no store, no hashing); an unsure answer must be true. + MayHaveOffers(ctx context.Context) bool + TryBundle(ctx context.Context, req PeerBundleRequest) (out PeerBundleOutcome, ok bool) + // HoldQueueItem reports whether the queue item came from a peer bundle and + // still awaits its full checks, which is when the ARRs must not be told. + HoldQueueItem(ctx context.Context, queueID int64) bool + // AttachQueueItem links freshly written bundle files to the queue item of a + // direct import, before its success handling runs. + AttachQueueItem(ctx context.Context, paths []string, queueID int64) error +} + +type peerFastPathHolder struct{ h PeerBundleFastPath } + +// peerItemInfo is the queue item's notification settings, parked for the +// processor (ProcessNzbFile does not receive the item). +type peerItemInfo struct { + targetPath string + skipArr bool +} + +// SetPeerBundleFastPath installs (or, with nil, removes) the fast path. +func (proc *Processor) SetPeerBundleFastPath(h PeerBundleFastPath) { + if h == nil { + proc.peerFastPath.Store(nil) + return + } + proc.peerFastPath.Store(&peerFastPathHolder{h: h}) +} + +func (proc *Processor) setPeerItem(queueID int, info peerItemInfo) { + proc.peerItems.Store(queueID, info) +} +func (proc *Processor) clearPeerItem(queueID int) { proc.peerItems.Delete(queueID) } + +// tryPeerBundle is the fast path, called with the NZB parsed and nothing else +// done. It never fails an import: any problem is "not taken". +// +// When it declines after building the NZB store, the store is returned so the +// import's own parse reuses it (see parser.ParseOptions.PrebuiltStore). +func (proc *Processor) tryPeerBundle(ctx context.Context, n *nzbparser.Nzb, filePath string, queueID int, virtualDir string, category *string, metadataJSON *string, downloadID *string) (result string, written []string, taken bool, prebuilt *prebuiltStore) { + holder := proc.peerFastPath.Load() + if holder == nil { + return "", nil, false, nil + } + defer func() { + if r := recover(); r != nil { + proc.log.ErrorContext(ctx, "Peer bundle fast path panicked; importing normally", "panic", r) + result, written, taken = "", nil, false + } + }() + // The common case: no peer offers anything. Nothing is hashed or built. + if !holder.h.MayHaveOffers(ctx) { + return "", nil, false, nil + } + store, index := metadata.BuildStore(n) + prebuilt = &prebuiltStore{store: store, index: index} + nzbName := proc.getCleanNzbName(filePath, queueID) + req := PeerBundleRequest{NZB: store, QueueID: queueID, NzbName: nzbName, TargetDir: filesystem.NzbFolderPath(virtualDir, nzbName)} + if category != nil { + req.Category = *category + } + if v, ok := proc.peerItems.Load(queueID); ok { + info := v.(peerItemInfo) + req.TargetPath, req.SkipArr = info.targetPath, info.skipArr + } + out, ok := holder.h.TryBundle(ctx, req) + if !ok { + return "", nil, false, prebuilt + } + if proc.recorder != nil { + nzbID := int64(queueID) + if err := proc.recorder.AddImportHistory(ctx, &database.ImportHistory{ + DownloadID: downloadID, NzbID: &nzbID, NzbName: nzbName, FileName: filepath.Base(out.ResultPath), + FileSize: out.TotalSize, VirtualPath: out.ResultPath, Category: category, Metadata: metadataJSON, + CompletedAt: time.Now(), + }); err != nil { + proc.log.ErrorContext(ctx, "Failed to add import history", "error", err, "nzb_name", nzbName) + } + } + proc.updateProgress(queueID, 100) + return out.ResultPath, out.WrittenPaths, true, nil +} + +// prebuiltStore is the NZB store (and message-id index) built for the peer +// lookup, kept for the import's parse when the NZB was not changed meanwhile. +type prebuiltStore struct { + store *metapb.NzbStore + index map[string]int64 +} + +// PeerProbeList is one file of a peer bundle for the release probe. +type PeerProbeList struct { + Name string + SegmentIDs []string +} + +var ( + // ErrPeerProbeMissing: sampled articles are missing on the providers. + ErrPeerProbeMissing = errors.New("importer: peer bundle probe found missing articles") + // ErrPeerProbeInconclusive: the providers could not answer. + ErrPeerProbeInconclusive = errors.New("importer: peer bundle probe inconclusive") +) + +// ProbePeerRelease runs the import's own release probe (a STAT sample against +// the user's providers, same sampling percentage and first/last-segment rules) +// over the segments of a peer bundle, plus the first and last segment of every +// file. It never downloads a body. +func (proc *Processor) ProbePeerRelease(ctx context.Context, files []PeerProbeList) error { + if !proc.poolManager.HasPool() { + return fmt.Errorf("%w: no NNTP provider", ErrPeerProbeInconclusive) + } + cfg := proc.configGetter() + all := make([]validation.FastFailFile, 0, len(files)) + edges := make([]validation.FastFailFile, 0, len(files)*2) + for _, f := range files { + segs := make([]*metapb.SegmentData, len(f.SegmentIDs)) + for i, id := range f.SegmentIDs { + segs[i] = &metapb.SegmentData{Id: id} + } + all = append(all, validation.FastFailFile{Filename: f.Name, Segments: segs}) + if len(segs) > 0 { + edges = append(edges, validation.FastFailFile{Filename: f.Name + "#first", Segments: segs[:1]}) + if len(segs) > 1 { + edges = append(edges, validation.FastFailFile{Filename: f.Name + "#last", Segments: segs[len(segs)-1:]}) + } + } + } + concurrency := proc.poolManager.StatSweepConcurrency(cfg.StatConcurrency()) + for _, probe := range []struct { + files []validation.FastFailFile + pct int + }{{all, cfg.Import.SegmentSamplePercentage}, {edges, 100}} { + verdict, err := validation.FastFailReleaseProbeVerdict(ctx, probe.files, proc.poolManager, probe.pct, + concurrency, proc.validationTimeout, proc.patchIndex, time.Time{}) + if err != nil { + return fmt.Errorf("%w: %w", ErrPeerProbeInconclusive, err) + } + if verdict.Missing { + return ErrPeerProbeMissing + } + } + return nil +} diff --git a/internal/importer/peer_bundle_service.go b/internal/importer/peer_bundle_service.go new file mode 100644 index 000000000..004bd2aa8 --- /dev/null +++ b/internal/importer/peer_bundle_service.go @@ -0,0 +1,278 @@ +package importer + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/importer/filesystem" +) + +// SetPeerBundleFastPath installs (or, with nil, removes) the peer-bundle fast +// path on the service and its processor. With none installed an import runs +// no peer-network code and no peer-network query. +func (s *Service) SetPeerBundleFastPath(h PeerBundleFastPath) { + s.mu.Lock() + if h == nil { + s.peerBundle = nil + } else { + s.peerBundle = h + } + s.mu.Unlock() + s.processor.SetPeerBundleFastPath(h) +} + +func (s *Service) peerBundleHook() PeerBundleFastPath { + s.mu.RLock() + defer s.mu.RUnlock() + return s.peerBundle +} + +func (s *Service) bundleState() *database.PeernetBundleImportRepository { + return database.NewPeernetBundleImportRepository(s.database.Connection(), s.database.Dialect()) +} + +// holdPeerQueueItem keeps a queue item whose files came from a peer bundle +// away from the ARRs until every file passed its full check. The persisted +// peer_hold mark (not the user's skip flag) keeps the item out of the +// SABnzbd-compatible history and shows it in the queue as work in progress; the +// in-memory skip flag stops this run's post-processing from notifying the ARRs. +// The mark survives a restart. It reports whether the item is held. +func (s *Service) holdPeerQueueItem(ctx context.Context, item *database.ImportQueueItem) bool { + h := s.peerBundleHook() + if h == nil || !h.HoldQueueItem(ctx, item.ID) { + return false + } + if err := s.bundleState().SetQueueHold(ctx, item.ID, true); err != nil { + s.log.ErrorContext(ctx, "Could not hold a peer bundle import back from the ARRs", "queue_id", item.ID, "error", err) + } + item.SkipArrNotification = true + return true +} + +// AnnounceVerifiedImport lifts the hold on a peer bundle import whose files all +// passed their full check and tells the ARRs, as a normal import would have at +// completion. The peer-bundle state guarantees it is called at most once per +// import. A queue item the user already removed is not announced. +func (s *Service) AnnounceVerifiedImport(ctx context.Context, queueID int64, category, targetPath, resultingPath string) error { + queued, err := s.database.Repository.GetQueueItem(ctx, queueID) + if err != nil { + return fmt.Errorf("read the queue item: %w", err) + } + if queued == nil { + return nil + } + if err := s.bundleState().SetQueueHold(ctx, queueID, false); err != nil { + return fmt.Errorf("lift the ARR hold: %w", err) + } + if s.broadcaster != nil { + s.broadcaster.BroadcastQueueChanged() + } + if queued.SkipArrNotification { + return nil // the user's own skip is honoured + } + item := &database.ImportQueueItem{ID: queueID} + if category != "" { + item.Category = &category + } + if targetPath != "" { + item.TargetPath = &targetPath + } + if err := s.postProcessor.NotifyARR(ctx, item, resultingPath); err != nil { + s.log.DebugContext(ctx, "ARR notification not sent for a verified peer import", "queue_id", queueID, "error", err) + } + return nil +} + +// FailImport turns the held queue item of a peer import that failed its full +// check into a failed one: the SABnzbd-compatible history then reports it as +// failed, so the ARR that grabbed it blocklists the release and searches again. +// A user's own skip keeps the item out of that history. +func (s *Service) FailImport(ctx context.Context, queueID int64, reason string) error { + changed, err := s.bundleState().FailHeldQueueItem(ctx, queueID, reason) + if err != nil { + return err + } + if changed && s.broadcaster != nil { + s.broadcaster.BroadcastQueueChanged() + } + return nil +} + +// ReleaseHold lifts the hold of a queue item no peer file refers to any more. +func (s *Service) ReleaseHold(ctx context.Context, queueID int64) error { + if err := s.bundleState().SetQueueHold(ctx, queueID, false); err != nil { + return err + } + if s.broadcaster != nil { + s.broadcaster.BroadcastQueueChanged() + } + return nil +} + +// RemoveFailedFile removes one file of a failed peer import the way the health +// worker removes a corrupted file when corruption handling is "delete": +// metadata (and with it the content index entry), the library copy and the +// health record. +func (s *Service) RemoveFailedFile(ctx context.Context, virtualPath string) error { + var physicalPath, rootPath string + if s.healthRepo != nil { + fh, err := s.healthRepo.GetFileHealth(ctx, virtualPath) + if err != nil { + s.log.WarnContext(ctx, "Could not read the health record of a failed peer file", "file_path", virtualPath, "error", err) + } + if fh != nil && fh.LibraryPath != nil { + physicalPath = *fh.LibraryPath + if cfg := s.configGetter(); cfg != nil { + rootPath = cfg.MountPath + if cfg.Health.LibraryDir != nil && *cfg.Health.LibraryDir != "" { + rootPath = *cfg.Health.LibraryDir + } + } + } + } + if err := s.metadataService.DeleteCorruptedFile(ctx, virtualPath, physicalPath, rootPath); err != nil { + return err + } + if s.healthRepo != nil { + if err := s.healthRepo.DeleteHealthRecord(ctx, virtualPath); err != nil { + return fmt.Errorf("delete the health record: %w", err) + } + } + return nil +} + +// ErrPeerImportNoNZB is returned for actions that need the NZB of a queue item +// whose release came from a peer bundle. +var ErrPeerImportNoNZB = errors.New("this release was imported from a peer bundle and has no NZB; remove the item and add the release again") + +// PeerImportRecord describes a finished direct import (one with no NZB). +type PeerImportRecord struct { + Title string + Category string + TargetPath string + ResultingPath string + WrittenPaths []string + TotalSize int64 + ReleaseID string + SkipArr bool + // Metadata is the queue metadata JSON of the import (media ids, as a + // Stremio-originated import stores them) and DownloadID its download id; + // both may be empty. + Metadata string + DownloadID string +} + +// RecordPeerImport gives a direct peer-bundle import the queue and history +// entries of a normal import and runs the normal success handling (VFS +// notification, symlinks, health scheduling, held ARR notification). It +// returns the queue id. +func (s *Service) RecordPeerImport(ctx context.Context, rec PeerImportRecord) (int64, error) { + name := sanitizeFilename(rec.Title) + if name == "" { + name = "release" + } + rid := rec.ReleaseID + if len(rid) > 16 { + rid = rid[:16] + } + // The path marks the row as a peer import (database.PeerImportNzbPrefix): + // no file is behind it. The row starts out completed, so a restart never + // takes it for interrupted work and retries it, and it is held at once. + item := &database.ImportQueueItem{ + NzbPath: database.PeerImportNzbPrefix + rid + "/" + name + ".nzb", + Priority: database.QueuePriorityNormal, + Status: database.QueueStatusCompleted, + MaxRetries: 1, + FileSize: &rec.TotalSize, + SkipArrNotification: rec.SkipArr, + } + if rec.Category != "" { + item.Category = &rec.Category + } + if rec.TargetPath != "" { + item.TargetPath = &rec.TargetPath + } + var metadata, downloadID *string + if rec.Metadata != "" { + metadata = &rec.Metadata + item.Metadata = metadata + } + if rec.DownloadID != "" { + downloadID = &rec.DownloadID + item.DownloadID = downloadID + } + if err := s.database.Repository.AddToQueue(ctx, item); err != nil { + return 0, fmt.Errorf("add peer import to the queue: %w", err) + } + nzbID := item.ID + // Roll the queue row back with the import when a later step fails: a + // completed row with no files would otherwise stay behind. + undo := func() { + bg := context.WithoutCancel(ctx) + _ = s.database.Repository.RemoveFromQueue(bg, nzbID) + _ = s.database.Repository.DeleteImportHistoryByNzbID(bg, nzbID) + } + if err := s.database.Repository.AddImportHistory(ctx, &database.ImportHistory{ + NzbID: &nzbID, NzbName: name + ".nzb", FileName: filepath.Base(rec.ResultingPath), FileSize: rec.TotalSize, + VirtualPath: rec.ResultingPath, Category: item.Category, CompletedAt: time.Now(), + Metadata: metadata, DownloadID: downloadID, + }); err != nil { + s.log.ErrorContext(ctx, "Failed to add import history for a peer import", "error", err) + } + if h := s.peerBundleHook(); h != nil { + if err := h.AttachQueueItem(ctx, rec.WrittenPaths, item.ID); err != nil { + undo() + return 0, fmt.Errorf("link peer import to its queue item: %w", err) + } + if h.HoldQueueItem(ctx, item.ID) { + if err := s.bundleState().SetQueueHold(ctx, item.ID, true); err != nil { + undo() + return 0, fmt.Errorf("hold the peer import: %w", err) + } + } + } + if err := s.handleProcessingSuccess(ctx, item, strings.TrimSuffix(rec.ResultingPath, "/"), rec.WrittenPaths); err != nil { + undo() + return 0, err + } + return item.ID, nil +} + +// ProbePeerRelease runs the import's release probe over a peer bundle's +// segments (see Processor.ProbePeerRelease). +func (s *Service) ProbePeerRelease(ctx context.Context, files []PeerProbeList) error { + return s.processor.ProbePeerRelease(ctx, files) +} + +// PeerImportDir is the virtual folder a direct peer import of the given +// category and title is written to: the one a queued NZB of that category and +// name would land in (the configured complete directory, the category's own +// folder, then the release folder), so a peer import and a normal import of +// the same release end up side by side. It returns "" for a title that cannot +// be turned into a plain folder name (see filesystem.SafeFolderName). +func (s *Service) PeerImportDir(category, title string) string { + base := "" + var reserved []string + if cfg := s.configGetter(); cfg != nil { + base = cfg.SABnzbd.CompleteDir + for _, c := range cfg.SABnzbd.Categories { + reserved = append(reserved, c.Name, filepath.Base(strings.TrimRight(c.Dir, "/"))) + } + } + name, ok := filesystem.SafeFolderName(title, reserved...) + if !ok { + return "" + } + item := &database.ImportQueueItem{NzbPath: filepath.Join(os.TempDir(), ".altmount-queue", "peer.nzb")} + if category != "" { + item.Category = &category + } + dir := s.calculateProcessVirtualDir(item, &base) + return strings.ReplaceAll(filepath.Join(dir, name), string(filepath.Separator), "/") +} diff --git a/internal/importer/peer_bundle_service_test.go b/internal/importer/peer_bundle_service_test.go new file mode 100644 index 000000000..4a8627335 --- /dev/null +++ b/internal/importer/peer_bundle_service_test.go @@ -0,0 +1,240 @@ +package importer + +import ( + "context" + "io" + "log/slog" + "os" + "path" + "path/filepath" + "testing" + + "github.com/kipsilabs/altmount/internal/config" + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/importer/filesystem" + "github.com/kipsilabs/altmount/internal/importer/postprocessor" + "github.com/kipsilabs/altmount/internal/metadata" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newPeerService(t *testing.T) *Service { + t.Helper() + db, err := database.NewDB(database.Config{DatabasePath: filepath.Join(t.TempDir(), "db")}) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + cfg := config.DefaultConfig() + getCfg := func() *config.Config { return cfg } + ms := metadata.NewMetadataService(t.TempDir()) + return &Service{ + database: db, metadataService: ms, configGetter: getCfg, + postProcessor: postprocessor.NewCoordinator(postprocessor.Config{ConfigGetter: getCfg, MetadataService: ms}), + log: slog.New(slog.NewTextHandler(io.Discard, nil)), + } +} + +func addItem(t *testing.T, s *Service, skip bool) *database.ImportQueueItem { + t.Helper() + item := &database.ImportQueueItem{NzbPath: "/x/a.nzb", Priority: database.QueuePriorityNormal, Status: database.QueueStatusProcessing, + MaxRetries: 1, SkipArrNotification: skip} + require.NoError(t, s.database.Repository.AddToQueue(context.Background(), item)) + return item +} + +func apiRepo(s *Service) *database.Repository { + return database.NewRepository(s.database.Connection(), s.database.Dialect()) +} + +func dbSkip(t *testing.T, s *Service, id int64) bool { + got, err := s.database.Repository.GetQueueItem(context.Background(), id) + require.NoError(t, err) + return got.SkipArrNotification +} + +func dbHold(t *testing.T, s *Service, id int64) bool { + got, err := apiRepo(s).GetQueueItem(context.Background(), id) + require.NoError(t, err) + require.NotNil(t, got) + return got.PeerHold +} + +func TestHoldPeerQueueItem(t *testing.T) { + ctx := context.Background() + s := newPeerService(t) + item := addItem(t, s, false) + + assert.False(t, s.holdPeerQueueItem(ctx, item), "no hook: nothing held") + assert.False(t, item.SkipArrNotification) + + s.peerBundle = &fakeFastPath{hold: map[int64]bool{item.ID + 1: true}} + assert.False(t, s.holdPeerQueueItem(ctx, item), "not a peer import") + assert.False(t, dbHold(t, s, item.ID)) + + s.peerBundle = &fakeFastPath{hold: map[int64]bool{item.ID: true}} + assert.True(t, s.holdPeerQueueItem(ctx, item)) + assert.True(t, item.SkipArrNotification, "the post-processing step sees the hold") + assert.True(t, dbHold(t, s, item.ID), "the hold is persisted, so it survives a restart") + assert.False(t, dbSkip(t, s, item.ID), "the user's skip flag is not used as the hold") +} + +func TestAnnounceVerifiedImportLiftsTheHold(t *testing.T) { + ctx := context.Background() + s := newPeerService(t) + item := addItem(t, s, false) + require.NoError(t, s.bundleState().SetQueueHold(ctx, item.ID, true)) + require.NoError(t, s.AnnounceVerifiedImport(ctx, item.ID, "movies", "", "movies/x")) + assert.False(t, dbHold(t, s, item.ID)) +} + +func TestAnnounceVerifiedImportHonoursTheUsersSkipAndAMissingItem(t *testing.T) { + ctx := context.Background() + s := newPeerService(t) + item := addItem(t, s, true) + require.NoError(t, s.bundleState().SetQueueHold(ctx, item.ID, true)) + require.NoError(t, s.AnnounceVerifiedImport(ctx, item.ID, "movies", "", "movies/x")) + assert.False(t, dbHold(t, s, item.ID)) + assert.True(t, dbSkip(t, s, item.ID), "the user's own skip survives the hold") + + // The user removed the queue item while the check was pending. + require.NoError(t, s.database.Repository.RemoveFromQueue(ctx, item.ID)) + require.NoError(t, s.AnnounceVerifiedImport(ctx, item.ID, "movies", "", "movies/x")) +} + +func TestFailImportTurnsAHeldItemIntoAFailedOne(t *testing.T) { + ctx := context.Background() + s := newPeerService(t) + item := addItem(t, s, false) + require.NoError(t, s.bundleState().SetQueueHold(ctx, item.ID, true)) + require.NoError(t, s.FailImport(ctx, item.ID, "gone bad")) + + got, err := apiRepo(s).GetQueueItem(ctx, item.ID) + require.NoError(t, err) + assert.Equal(t, database.QueueStatusFailed, got.Status) + require.NotNil(t, got.ErrorMessage) + assert.Equal(t, "gone bad", *got.ErrorMessage) + assert.False(t, got.PeerHold) + require.NoError(t, s.FailImport(ctx, item.ID, "again"), "idempotent") + require.NoError(t, s.FailImport(ctx, 9999, "no such item"), "a removed item is fine") +} + +func TestRecordPeerImportCreatesQueueAndHistoryEntries(t *testing.T) { + ctx := context.Background() + s := newPeerService(t) + fp := &fakeFastPath{hold: map[int64]bool{1: true}} + s.peerBundle = fp + id, err := s.RecordPeerImport(ctx, PeerImportRecord{Title: "My Release", Category: "movies", ResultingPath: "movies/My Release", + WrittenPaths: []string{"movies/My Release/a.mkv"}, TotalSize: 99, ReleaseID: "abcdef0123456789abcdef"}) + require.NoError(t, err) + got, err := s.database.Repository.GetQueueItem(ctx, id) + require.NoError(t, err) + assert.Equal(t, database.QueueStatusCompleted, got.Status) + assert.Equal(t, "movies", *got.Category) + assert.False(t, got.SkipArrNotification, "the user's skip flag is not the hold") + assert.True(t, dbHold(t, s, id), "held from the ARRs until the full check passed") + rows, err := database.NewRepository(s.database.Connection(), s.database.Dialect()).ListSABnzbdHistory(ctx, "", 10, 0) + require.NoError(t, err) + assert.Empty(t, rows, "a held import is not in the SABnzbd-compatible history") + + require.NoError(t, s.AnnounceVerifiedImport(ctx, id, "movies", "", "movies/My Release")) + rows, err = database.NewRepository(s.database.Connection(), s.database.Dialect()).ListSABnzbdHistory(ctx, "", 10, 0) + require.NoError(t, err) + require.Len(t, rows, 1, "visible to the ARRs once verified") + assert.Contains(t, rows[0].Name, "My Release") +} + +// The queue row of a direct import has no NZB behind it. It must survive a +// restart as it is (not be taken for interrupted work and retried into a +// failure), and no action that needs the NZB may run on it. +func TestRecordPeerImportRowIsRestartSafeAndNotRetryable(t *testing.T) { + ctx := context.Background() + s := newPeerService(t) + var midStatus database.QueueStatus + var midPeer bool + fp := &fakeFastPath{hold: map[int64]bool{1: true}} + // A crash between AddToQueue and the end of the success handling leaves the + // row as it is at this point; the restart recovery must not touch it. + fp.onAttach = func(id int64) { + require.NoError(t, s.database.Repository.ResetStaleItems(ctx)) + got, err := s.database.Repository.GetQueueItem(ctx, id) + require.NoError(t, err) + midStatus, midPeer = got.Status, got.IsPeerImport() + } + s.peerBundle = fp + id, err := s.RecordPeerImport(ctx, PeerImportRecord{Title: "My Release", Category: "movies", ResultingPath: "movies/My Release", + WrittenPaths: []string{"movies/My Release/a.mkv"}, TotalSize: 99, ReleaseID: "abcdef0123456789abcdef"}) + require.NoError(t, err) + assert.True(t, midPeer) + assert.Equal(t, database.QueueStatusCompleted, midStatus, "a restart mid-import does not reset the row to pending") + + got, err := s.database.Repository.GetQueueItem(ctx, id) + require.NoError(t, err) + assert.True(t, got.IsPeerImport()) + + assert.ErrorIs(t, s.ExecuteItem(ctx, id), ErrPeerImportNoNZB) + got, _ = s.database.Repository.GetQueueItem(ctx, id) + assert.Equal(t, database.QueueStatusCompleted, got.Status, "a refused retry changes nothing") +} + +func TestRecordPeerImportRollsTheQueueRowBackOnFailure(t *testing.T) { + ctx := context.Background() + s := newPeerService(t) + s.peerBundle = &fakeFastPath{attach: assert.AnError} + _, err := s.RecordPeerImport(ctx, PeerImportRecord{Title: "My Release", Category: "movies", ResultingPath: "movies/My Release", + WrittenPaths: []string{"movies/My Release/a.mkv"}, TotalSize: 99, ReleaseID: "abcdef0123456789abcdef"}) + require.Error(t, err) + items, err := apiRepo(s).ListQueueItems(ctx, nil, "", "", 10, 0, "updated_at", "desc") + require.NoError(t, err) + assert.Empty(t, items, "no completed row without files is left behind") +} + +func TestRecordPeerImportKeepsMediaMetadataAndDownloadID(t *testing.T) { + ctx := context.Background() + s := newPeerService(t) + meta := `{"imdb_id":"tt0111161","type":"movie"}` + dl := "stremio:abc" + id, err := s.RecordPeerImport(ctx, PeerImportRecord{Title: "My Release", Category: "movies", ResultingPath: "movies/My Release", + WrittenPaths: []string{"movies/My Release/a.mkv"}, TotalSize: 99, ReleaseID: "abcdef0123456789abcdef", Metadata: meta, DownloadID: dl}) + require.NoError(t, err) + got, err := s.database.Repository.GetQueueItem(ctx, id) + require.NoError(t, err) + require.NotNil(t, got.Metadata) + assert.Equal(t, meta, *got.Metadata) + require.NotNil(t, got.DownloadID) + assert.Equal(t, dl, *got.DownloadID) + o, err := database.NewPeernetOriginRepository(s.database.Connection(), s.database.Dialect()).OriginForPath(ctx, "movies/My Release/a.mkv") + require.NoError(t, err) + assert.Equal(t, meta, o.QueueMetadata, "the record published for this release carries the media ids") +} + +func TestPeerImportDirFollowsTheNormalImportRules(t *testing.T) { + s := newPeerService(t) + cfg := s.configGetter() + cfg.SABnzbd.CompleteDir = "/complete" + for _, category := range []string{"Movies", "TV", "no-such-category"} { + // The virtual folder a queued NZB of this category and title lands in. + item := &database.ImportQueueItem{NzbPath: filepath.Join(os.TempDir(), ".altmount-queue", "x.nzb"), Category: &category} + base := cfg.SABnzbd.CompleteDir + want := filesystem.NzbFolderPath(s.calculateProcessVirtualDir(item, &base), "My.Title.2020.1080p.nzb") + got := s.PeerImportDir(category, "My.Title.2020.1080p") + assert.Equal(t, want, got, category) + assert.Contains(t, got, "My.Title.2020.1080p") + } + assert.NotEqual(t, s.PeerImportDir("Movies", "A"), s.PeerImportDir("TV", "A"), "the category decides the folder") +} + +func TestPeerImportDirRefusesTitlesThatEscapeTheCategoryFolder(t *testing.T) { + s := newPeerService(t) + cfg := s.configGetter() + cfg.SABnzbd.CompleteDir = "/complete" + parent := s.PeerImportDir("Movies", "ok") + catDir := path.Dir(parent) + for _, title := range []string{"...mkv", ".mkv", "..", ".", "", ".hidden", "..mkv.nzb", ".nzbs", "patches", "corrupted_metadata", "_migrated", ".peernet-staging", "PATCHES"} { + got := s.PeerImportDir("Movies", title) + if got != "" { + assert.Equal(t, catDir, path.Dir(got), "title %q must stay one level below the category folder, got %q", title, got) + assert.NotContains(t, []string{".", "..", ""}, path.Base(got)) + } + assert.Empty(t, got, "title %q must be refused", title) + } + assert.Equal(t, catDir+"/Some Movie", s.PeerImportDir("Movies", "Some Movie.mkv")) +} diff --git a/internal/importer/peer_bundle_test.go b/internal/importer/peer_bundle_test.go new file mode 100644 index 000000000..c7ef52404 --- /dev/null +++ b/internal/importer/peer_bundle_test.go @@ -0,0 +1,227 @@ +package importer + +import ( + "context" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/javi11/nntppool/v5" + "github.com/javi11/nzbparser" + "github.com/kipsilabs/altmount/internal/database" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/testsupport/fakepool" + "github.com/kipsilabs/altmount/internal/testsupport/nzbbuild" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeFastPath struct { + mu sync.Mutex + reqs []PeerBundleRequest + take bool + panics bool + env *batteryEnv + hold map[int64]bool + attach error + noOffers bool + offerAsks atomic.Int32 + // onAttach runs while the queue row exists but the import is unfinished. + onAttach func(queueID int64) +} + +func (f *fakeFastPath) TryBundle(_ context.Context, req PeerBundleRequest) (PeerBundleOutcome, bool) { + f.mu.Lock() + f.reqs = append(f.reqs, req) + f.mu.Unlock() + if f.panics { + panic("boom") + } + if !f.take { + return PeerBundleOutcome{}, false + } + p := req.TargetDir + "/movie.mkv" + _ = f.env.svc.WriteFileMetadata(p, &metapb.FileMetadata{FileSize: 10, Status: metapb.FileStatus_FILE_STATUS_HEALTHY}) + return PeerBundleOutcome{ResultPath: req.TargetDir, WrittenPaths: []string{p}, TotalSize: 10}, true +} + +func (f *fakeFastPath) MayHaveOffers(context.Context) bool { + f.offerAsks.Add(1) + return !f.noOffers +} + +func (f *fakeFastPath) AttachQueueItem(_ context.Context, _ []string, id int64) error { + if f.onAttach != nil { + f.onAttach(id) + } + return f.attach +} + +func (f *fakeFastPath) HoldQueueItem(_ context.Context, id int64) bool { return f.hold[id] } + +type memRecorder struct{ rows []*database.ImportHistory } + +func (r *memRecorder) AddImportHistory(_ context.Context, h *database.ImportHistory) error { + r.rows = append(r.rows, h) + return nil +} + +func missingBehavior() fakepool.SegmentBehavior { + return fakepool.SegmentBehavior{Err: nntppool.ErrArticleNotFound} +} + +// registerContentWithLatency is registerContent with a per-article delay. +func (e *batteryEnv) registerContentWithLatency(prefix string, content []byte, partSize int, latency time.Duration) []nzbbuild.Segment { + segs := e.registerContent(prefix, content, partSize, 1.0, nil) + for i, s := range segs { + end := min((i+1)*partSize, len(content)) + e.client.SetBehavior(s.ID, fakepool.SegmentBehavior{Bytes: content[i*partSize : end], Latency: latency}) + } + return segs +} + +func cleanNZB(e *batteryEnv) (nzbPath string) { + content := make([]byte, 4096) + for i := range content { + content[i] = byte(i % 251) + } + segs := e.registerContent("fp", content, 1024, 1.0, nil) + return nzbbuild.WriteTemp(e.t, nzbbuild.Build(nzbbuild.File{Subject: "movie.mkv", Segments: segs}), "Some.Release") +} + +func TestPeerFastPath_TakenSkipsProbeAndAnalysisAndRecordsHistory(t *testing.T) { + e := newBatteryEnv(t) + rec := &memRecorder{} + e.proc.SetRecorder(rec) + fp := &fakeFastPath{take: true, env: e} + e.proc.SetPeerBundleFastPath(fp) + nzbPath := cleanNZB(e) + + result, written, err := e.proc.ProcessNzbFile(context.Background(), nzbPath, filepath.Dir(nzbPath), 5, nil, nil, nil, nil, nil, nil) + require.NoError(t, err) + assert.Equal(t, "/Some.Release", result) + assert.Equal(t, []string{"/Some.Release/movie.mkv"}, written) + assert.Zero(t, e.client.StatCalls(), "no release probe") + assert.Zero(t, e.client.BodyCalls(), "no article download") + require.Len(t, fp.reqs, 1) + assert.Equal(t, 5, fp.reqs[0].QueueID) + assert.Equal(t, "/Some.Release", fp.reqs[0].TargetDir) + require.Len(t, fp.reqs[0].NZB.Files, 1) + assert.Len(t, fp.reqs[0].NZB.Files[0].Segments, 4, "the parsed NZB's segment list is handed over") + require.Len(t, rec.rows, 1) + assert.Equal(t, "Some.Release.nzb", rec.rows[0].NzbName) + assert.Equal(t, "/Some.Release", rec.rows[0].VirtualPath) +} + +// The normal import result must be identical whether no fast path is installed +// or one is installed that declines, errors out or panics. +func TestPeerFastPath_FallbackLeavesTheNormalImportIdentical(t *testing.T) { + run := func(install func(e *batteryEnv)) (string, []string, []byte) { + e := newBatteryEnv(t) + install(e) + nzbPath := cleanNZB(e) + res, written, err := e.proc.ProcessNzbFile(context.Background(), nzbPath, filepath.Dir(nzbPath), 5, nil, nil, nil, nil, nil, nil) + require.NoError(t, err) + meta, err := e.svc.ReadFileMetadata(written[0]) + require.NoError(t, err) + return res, written, []byte(meta.OriginalFilename + "|" + filepath.Base(written[0])) + } + baseRes, baseWritten, baseMeta := run(func(*batteryEnv) {}) + for name, install := range map[string]func(e *batteryEnv){ + "declines": func(e *batteryEnv) { e.proc.SetPeerBundleFastPath(&fakeFastPath{env: e}) }, + "panics": func(e *batteryEnv) { e.proc.SetPeerBundleFastPath(&fakeFastPath{env: e, panics: true}) }, + "no offers": func(e *batteryEnv) { e.proc.SetPeerBundleFastPath(&fakeFastPath{env: e, noOffers: true}) }, + } { + t.Run(name, func(t *testing.T) { + res, written, meta := run(install) + assert.Equal(t, baseRes, res) + assert.Equal(t, baseWritten, written) + assert.Equal(t, baseMeta, meta) + }) + } +} + +// With no offer on this node the import asks one cheap question and does +// nothing else for the peer network: no store is built for it, nothing is handed +// to the hook. +func TestPeerFastPath_NoOffersMeansNoPeerWork(t *testing.T) { + e := newBatteryEnv(t) + fp := &fakeFastPath{take: true, noOffers: true, env: e} + e.proc.SetPeerBundleFastPath(fp) + nzbPath := cleanNZB(e) + n := parseTestNzb(t, nzbPath) + + result, written, taken, prebuilt := e.proc.tryPeerBundle(context.Background(), n, nzbPath, 5, "/", nil, nil, nil) + assert.False(t, taken) + assert.Empty(t, result) + assert.Empty(t, written) + assert.Nil(t, prebuilt, "no store was built") + assert.Equal(t, int32(1), fp.offerAsks.Load()) + assert.Empty(t, fp.reqs, "the hook never saw the NZB") + + // Control: with offers the store is built once and kept for the parse. + fp.noOffers, fp.take = false, false + _, _, taken, prebuilt = e.proc.tryPeerBundle(context.Background(), n, nzbPath, 5, "/", nil, nil, nil) + assert.False(t, taken) + require.NotNil(t, prebuilt) + assert.Same(t, prebuilt.store, fp.reqs[0].NZB, "the store handed to the hook is the one kept for the parse") +} + +func TestPeerFastPath_RemovedHookIsNeverCalled(t *testing.T) { + e := newBatteryEnv(t) + fp := &fakeFastPath{take: true, env: e} + e.proc.SetPeerBundleFastPath(fp) + e.proc.SetPeerBundleFastPath(nil) + nzbPath := cleanNZB(e) + _, _, err := e.proc.ProcessNzbFile(context.Background(), nzbPath, filepath.Dir(nzbPath), 5, nil, nil, nil, nil, nil, nil) + require.NoError(t, err) + assert.Empty(t, fp.reqs) +} + +func TestProbePeerRelease(t *testing.T) { + e := newBatteryEnv(t) + ids := []string{"a1@x", "a2@x", "a3@x", "a4@x", "a5@x", "a6@x"} + require.NoError(t, e.proc.ProbePeerRelease(context.Background(), []PeerProbeList{{Name: "f.mkv", SegmentIDs: ids}})) + assert.Positive(t, e.client.StatCalls(), "a STAT sample was taken") + assert.Zero(t, e.client.BodyCalls(), "never a body") + + e2 := newBatteryEnv(t) + e2.client.SetBehavior("a6@x", missingBehavior()) + err := e2.proc.ProbePeerRelease(context.Background(), []PeerProbeList{{Name: "f.mkv", SegmentIDs: ids}}) + assert.ErrorIs(t, err, ErrPeerProbeMissing, "the last segment of every file is always checked") +} + +// The saved time on a synthetic archive import: every article costs 15 ms, as +// on a real provider. The fast path skips the probe, the first-segment warm-up +// and the archive analysis. +func TestPeerFastPath_TimeSavedOnArchiveImport(t *testing.T) { + timed := func(withFastPath bool) time.Duration { + e := newBatteryEnv(t) + rarBytes := loadFixture(t, filepath.Join("rar_single", "archive.rar")) + segs := e.registerContentWithLatency("tm", rarBytes, archivePartSize, 15*time.Millisecond) + nzbPath := nzbbuild.WriteTemp(t, nzbbuild.Build(nzbbuild.File{Subject: "archive.rar", Segments: segs}), "archive") + if withFastPath { + e.proc.SetPeerBundleFastPath(&fakeFastPath{take: true, env: e}) + } + start := time.Now() + _, _, err := e.proc.ProcessNzbFile(context.Background(), nzbPath, filepath.Dir(nzbPath), 1, nil, nil, nil, nil, nil, nil) + require.NoError(t, err) + return time.Since(start) + } + normal, fast := timed(false), timed(true) + t.Logf("archive import with 15ms/article: normal %v, bundle fast path %v (saved %v)", normal, fast, normal-fast) + assert.Less(t, fast, normal) +} + +func parseTestNzb(t *testing.T, path string) *nzbparser.Nzb { + t.Helper() + f, err := os.Open(path) + require.NoError(t, err) + defer f.Close() + n, err := nzbparser.Parse(f) + require.NoError(t, err) + return n +} diff --git a/internal/importer/postprocessor/health_scheduler.go b/internal/importer/postprocessor/health_scheduler.go index 42e7244e4..3b6b5a39a 100644 --- a/internal/importer/postprocessor/health_scheduler.go +++ b/internal/importer/postprocessor/health_scheduler.go @@ -17,6 +17,16 @@ import ( // ISO expansion), so this is a safety net, not an expected limit. const maxDirExpansionDepth = 8 +// WillScheduleHealth reports whether an import under the current configuration +// creates a health record (and so a check) for the written file at path. It is +// the single rule ScheduleHealthCheck applies and the peer-bundle importer asks. +func WillScheduleHealth(cfg *config.Config, path string) bool { + if cfg == nil || !cfg.GetHealthEnabled() { + return false + } + return cfg.Import.ImportStrategy == config.ImportStrategyNone || isArrImportableMedia(path) +} + // ScheduleHealthCheck schedules an immediate health check for an imported file. // Multi-file imports (e.g. season packs) schedule one check per written virtual // file — resultingPath is a directory there, so the per-file paths are the only @@ -55,7 +65,7 @@ func (c *Coordinator) ScheduleHealthCheck(ctx context.Context, item *database.Im if cfg.Import.ImportStrategy != config.ImportStrategyNone { media := make([]string, 0, len(paths)) for _, p := range paths { - if isArrImportableMedia(p) { + if WillScheduleHealth(cfg, p) { media = append(media, p) } } @@ -163,6 +173,7 @@ func isArrImportableMedia(p string) bool { } return arrAudioBookExtensions[strings.ToLower(path.Ext(p))] } + // expandWrittenPaths resolves "DIR:"-prefixed entries (whole-directory imports such // as RAR/7z archives, which only report their NZB folder) into the per-file virtual // paths beneath them by walking the metadata tree. Plain file entries pass through diff --git a/internal/importer/processor.go b/internal/importer/processor.go index 280427719..c83efd1ad 100644 --- a/internal/importer/processor.go +++ b/internal/importer/processor.go @@ -11,6 +11,7 @@ import ( "regexp" "strings" "sync" + "sync/atomic" "time" "github.com/javi11/nntppool/v5" @@ -54,8 +55,11 @@ type Processor struct { log *slog.Logger broadcaster *progress.ProgressBroadcaster // WebSocket progress broadcaster recorder HistoryRecorder - par2Repair RepairEnqueuer // optional; queues PAR2 repairs at import time - patchIndex validation.PatchIndex // optional; locally repaired articles count as available + par2Repair RepairEnqueuer // optional; queues PAR2 repairs at import time + patchIndex validation.PatchIndex // optional; locally repaired articles count as available + peerFastPath atomic.Pointer[peerFastPathHolder] // optional; nil unless the peer network is on + pwRecorder atomic.Pointer[nzbPasswordHolder] // optional; nil unless the peer network is on + peerItems sync.Map // queue id -> peerItemInfo // Pre-compiled regex patterns for RAR file sorting rarPartPattern *regexp.Regexp // pattern.part###.rar @@ -1010,6 +1014,25 @@ func (proc *Processor) ProcessNzbFile(ctx context.Context, filePath, relativePat if len(n.Files) == 0 { return "", nil, NewNonRetryableError("NZB file contains no files", nil) } + // Peer-bundle fast path (nil unless the peer network is on): with the + // NZB parsed and nothing else done, a verified bundle of this exact + // release can replace the probe, warm-up and archive analysis below. + // Any problem falls through to the normal import. + var prebuilt *prebuiltStore + if proc.peerFastPath.Load() != nil { + vd := "" + if virtualDirOverride != nil { + vd = *virtualDirOverride + } else { + vd = filesystem.CalculateVirtualDirectory(filePath, relativePath) + } + var result string + var written []string + var taken bool + if result, written, taken, prebuilt = proc.tryPeerBundle(ctx, n, filePath, queueID, vd, category, metadata, downloadID); taken { + return result, written, nil + } + } // Resolution atomically replaces this path; release the original handle // first so the replacement works on Windows as well. if err := file.Close(); err != nil { @@ -1028,7 +1051,8 @@ func (proc *Processor) ProcessNzbFile(ctx context.Context, filePath, relativePat // bytes keep their offsets and the gaps are served as holes. Done // before the fast-fail sweep so those known misses count without a // STAT and before the store is built so they hold a store index. - if gaps := parser.InsertSegmentGapPlaceholders(n); gaps > 0 && proc.log != nil { + gaps := parser.InsertSegmentGapPlaceholders(n) + if gaps > 0 && proc.log != nil { proc.log.InfoContext(ctx, "NZB omits articles; inserted gap placeholders", "placeholders", gaps, "file_path", filePath) } @@ -1074,10 +1098,18 @@ func (proc *Processor) ProcessNzbFile(ctx context.Context, filePath, relativePat } parseTracker := progress.NewTracker(proc.broadcaster, queueID, 2, 10) - parsed, err = proc.parser.ParseNzb(ctx, n, filePath, parseTracker, parser.ParseOptions{ + parseOpts := parser.ParseOptions{ BrokenFileIndexes: brokenIdx, KnownMissingSegmentIDs: missingIDs, - }) + } + // The store built for the peer lookup describes this NZB exactly + // unless gap placeholders were inserted into it (a nested NZB, parsed + // afresh below, never reuses it). + if prebuilt != nil && gaps == 0 { + parseOpts.PrebuiltStore, parseOpts.PrebuiltIndex = prebuilt.store, prebuilt.index + } + prebuilt = nil + parsed, err = proc.parser.ParseNzb(ctx, n, filePath, parseTracker, parseOpts) if err != nil { return "", nil, NewNonRetryableError("failed to parse NZB file", err) } @@ -1343,6 +1375,7 @@ func (proc *Processor) ProcessNzbFile(ctx context.Context, filePath, relativePat // release's recovery volumes are most likely still retrievable. proc.persistKnownHoles(ctx, writtenPaths, degradedFiles, missingIDs, cfg.ProviderFingerprint()) proc.queueImportRepairs(ctx, repairOnImport(cfg, category, downloadID), writtenPaths, degradedFiles) + proc.recordNZBPassword(ctx, parsed, metadata, writtenPaths) proc.updateProgress(queueID, 100) } else if repairEnabled, hasPar2 := repairOnImport(cfg, category, downloadID), len(par2Files) > 0; shouldDeferCorruptArchive(err, repairEnabled, hasPar2) || shouldDeferMissingArchive(err, repairEnabled, hasPar2) { // Corrupt-but-present articles, or an article the fast-fail probe's diff --git a/internal/importer/service.go b/internal/importer/service.go index 819b3ba9a..e7b8538b8 100644 --- a/internal/importer/service.go +++ b/internal/importer/service.go @@ -258,6 +258,7 @@ type Service struct { broadcaster *progress.ProgressBroadcaster // WebSocket progress broadcaster userRepo *database.UserRepository // User repository for API key lookup poolManager pool.Manager // Pool manager — used to push admission caps on config change + peerBundle PeerBundleFastPath // optional; nil unless the peer network is on contentVerifyFS contentverify.Opener // Optional: real NzbFilesystem, set post-construction to break the init-order cycle with nzbfilesystem log *slog.Logger @@ -468,7 +469,7 @@ func (s *Service) Resume() { // IsPaused returns whether the service is paused func (s *Service) IsPaused() bool { - return s.queueManager.IsPaused() + return s.queueManager != nil && s.queueManager.IsPaused() } func (s *Service) RegisterConfigChangeHandler(configManager any) { @@ -938,6 +939,15 @@ func (s *Service) processNzbItem(ctx context.Context, item *database.ImportQueue } } + if s.peerBundleHook() != nil { + var targetPath string + if item.TargetPath != nil { + targetPath = *item.TargetPath + } + s.processor.setPeerItem(int(item.ID), peerItemInfo{targetPath: targetPath, skipArr: item.SkipArrNotification}) + defer s.processor.clearPeerItem(int(item.ID)) + } + resultPath, writtenPaths, err := s.processor.ProcessNzbFile(ctx, item.NzbPath, basePath, int(item.ID), allowedExtensionsOverride, &virtualDir, extractedFiles, item.Category, item.Metadata, item.DownloadID) var expansion *nestedNzbJobsError if errors.As(err, &expansion) { @@ -1396,6 +1406,10 @@ func (s *Service) handleProcessingSuccess(ctx context.Context, item *database.Im s.broadcaster.NotifyStreamable(int(item.ID), resultingPath) } + // A peer-bundle import is playable at once but is not announced to the ARRs + // until every file passed its full check. + s.holdPeerQueueItem(ctx, item) + // Refresh mount path if needed before post-processing s.postProcessor.RefreshMountPathIfNeeded(ctx, resultingPath, item.ID) @@ -1844,6 +1858,9 @@ func (s *Service) CancelProcessing(itemID int64) error { // ExecuteItem manually triggers processing for a specific queue item, bypassing concurrency limits. func (s *Service) ExecuteItem(ctx context.Context, itemID int64) error { + if item, err := s.database.Repository.GetQueueItem(ctx, itemID); err == nil && item.IsPeerImport() { + return ErrPeerImportNoNZB + } s.ProcessItemInBackground(ctx, itemID) return nil } diff --git a/internal/metadata/service.go b/internal/metadata/service.go index d14cebd9a..3b7d0f635 100644 --- a/internal/metadata/service.go +++ b/internal/metadata/service.go @@ -97,6 +97,9 @@ type MetadataService struct { // renamed or deleted. nil disables notifications. Held atomically so it // can be set and cleared while the service is in use. observer atomic.Pointer[observerBox] + // secondary is a second, independent observer (the peer-bundle state keeps + // its rows consistent with deletes and renames). Same rules as observer. + secondary atomic.Pointer[observerBox] } // observerBox lets an interface value live in an atomic.Pointer. @@ -104,12 +107,51 @@ type observerBox struct{ o FileObserver } // fileObserver returns the current observer, or nil. func (ms *MetadataService) fileObserver() FileObserver { - if b := ms.observer.Load(); b != nil { - return b.o + p, q := ms.observer.Load(), ms.secondary.Load() + switch { + case p != nil && q != nil: + return observerPair{p.o, q.o} + case p != nil: + return p.o + case q != nil: + return q.o } return nil } +// observerPair forwards every notification to both observers. +type observerPair struct{ a, b FileObserver } + +func (p observerPair) FileWritten(ctx context.Context, v string) { + p.a.FileWritten(ctx, v) + p.b.FileWritten(ctx, v) +} + +func (p observerPair) FileDeleted(ctx context.Context, v string) { + p.a.FileDeleted(ctx, v) + p.b.FileDeleted(ctx, v) +} + +func (p observerPair) DirectoryDeleted(ctx context.Context, v string) { + p.a.DirectoryDeleted(ctx, v) + p.b.DirectoryDeleted(ctx, v) +} + +func (p observerPair) DirectoryRenamed(ctx context.Context, from, to string) { + p.a.DirectoryRenamed(ctx, from, to) + p.b.DirectoryRenamed(ctx, from, to) +} + +// SetSecondaryFileObserver wires in a second observer next to SetFileObserver's. +// Passing nil removes it. +func (ms *MetadataService) SetSecondaryFileObserver(o FileObserver) { + if o == nil { + ms.secondary.Store(nil) + return + } + ms.secondary.Store(&observerBox{o: o}) +} + // NewMetadataService creates a new metadata service func NewMetadataService(rootPath string) *MetadataService { liteCache, _ := lru.New[string, *FileMetadataLite](defaultMetadataCacheSize) diff --git a/internal/peernet/bridge/bundleimport.go b/internal/peernet/bridge/bundleimport.go new file mode 100644 index 000000000..4669217c3 --- /dev/null +++ b/internal/peernet/bridge/bundleimport.go @@ -0,0 +1,270 @@ +package bridge + +import ( + "context" + "errors" + "log/slog" + "strings" + "time" + + "github.com/kipsilabs/altmount/internal/importer" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/bundleimport" + "github.com/kipsilabs/altmount/internal/peernet/contentid" + "github.com/kipsilabs/altmount/internal/peernet/service" +) + +// DefaultFastPathFetchTimeout is how long the importer waits for a bundle +// before it imports normally. It is the most the fast path can ever add to an +// import. +const DefaultFastPathFetchTimeout = 5 * time.Second + +// DefaultFastPathAcceptTimeout is how long verifying a fetched layout may take +// before the fast path gives up and the NZB is imported normally. +const DefaultFastPathAcceptTimeout = 15 * time.Second + +// minOpenCorroborators is how many distinct network ranges must announce the +// same Bundle ID before an open-network layout may replace archive analysis. +const minOpenCorroborators = 2 + +// BundleAcceptor is the part of *bundleimport.Importer the importer fast path +// needs. +type BundleAcceptor interface { + Accept(ctx context.Context, req bundleimport.Request) (*bundleimport.Result, error) + QueueHasPending(ctx context.Context, queueID int64) bool + AttachQueue(ctx context.Context, paths []string, queueID int64) error +} + +// FastPathConfig configures NewPeerBundleFastPath. +type FastPathConfig struct { + Acceptor BundleAcceptor + Fetcher bundleimport.Fetcher + // Enabled says the peer network is on with at least one network at level 1 + // or above. It is read on every import; false means the fast path does + // nothing at all. + Enabled func() bool + // OpenFastPath says the user allows layouts offered on the open network to + // replace archive analysis (peernet.open_network.fast_path). Offers from a + // private network need no switch. Nil means not allowed. + OpenFastPath func() bool + // AnyOffers says whether this node holds any bundle offer at all. It must be + // cheap and cached (it is asked on every import); false makes the fast path + // do nothing. Nil means "unknown": offers are assumed. + AnyOffers func(ctx context.Context) bool + // AcceptTimeout bounds the accept pipeline of the fast path (release probe, + // sampled header reads, media structure check). A layout that cannot be + // verified in time is declined and the normal import runs (zero: + // DefaultFastPathAcceptTimeout). + AcceptTimeout time.Duration + // FetchTimeout bounds the fetch (zero: DefaultFastPathFetchTimeout). + FetchTimeout time.Duration +} + +type fastPath struct{ cfg FastPathConfig } + +// NewPeerBundleFastPath returns the importer's fast path hook. A nil acceptor +// or fetcher yields nil, so the importer then has no hook at all. +func NewPeerBundleFastPath(cfg FastPathConfig) importer.PeerBundleFastPath { + if cfg.Acceptor == nil || cfg.Fetcher == nil || cfg.Enabled == nil { + return nil + } + if cfg.FetchTimeout <= 0 { + cfg.FetchTimeout = DefaultFastPathFetchTimeout + } + if cfg.AcceptTimeout <= 0 { + cfg.AcceptTimeout = DefaultFastPathAcceptTimeout + } + return &fastPath{cfg: cfg} +} + +// encodedSizes returns the NZB's yEnc-encoded size of every segment, in the +// shape canonicalIDs returns. +func encodedSizes(store *metapb.NzbStore) [][]int64 { + out := make([][]int64, 0, len(store.GetFiles())) + for _, f := range store.GetFiles() { + sizes := make([]int64, 0, len(f.GetSegments())) + for _, s := range f.GetSegments() { + sizes = append(sizes, s.GetBytes()) + } + out = append(out, sizes) + } + return out +} + +func canonicalIDs(store *metapb.NzbStore) [][]string { + out := make([][]string, 0, len(store.GetFiles())) + for _, f := range store.GetFiles() { + ids := make([]string, 0, len(f.GetSegments())) + for _, s := range f.GetSegments() { + id := strings.TrimSuffix(strings.TrimPrefix(s.GetId(), "<"), ">") + ids = append(ids, id) + } + out = append(out, ids) + } + return out +} + +// TryBundle asks the network for the bundle of the exact release the user's +// NZB describes and, if a verified one arrives in time and survives the accept +// pipeline, imports it. Every failure is "not taken". +func (f *fastPath) TryBundle(ctx context.Context, req importer.PeerBundleRequest) (importer.PeerBundleOutcome, bool) { + if !f.cfg.Enabled() || req.NZB == nil { + return importer.PeerBundleOutcome{}, false + } + rid, err := contentid.ReleaseID(req.NZB) + if err != nil { + return importer.PeerBundleOutcome{}, false + } + fetchCtx, cancel := context.WithTimeout(ctx, f.cfg.FetchTimeout) + b, origin, err := f.cfg.Fetcher.FetchBundle(fetchCtx, rid.String()) + cancel() + if err != nil || b == nil || b.Scheme != bundle.SchemeNZB || b.ReleaseID != rid { + return importer.PeerBundleOutcome{}, false + } + if origin.Open && (f.cfg.OpenFastPath == nil || !f.cfg.OpenFastPath() || origin.Corroborators < minOpenCorroborators) { + return importer.PeerBundleOutcome{}, false + } + acceptCtx, cancelAccept := context.WithTimeout(ctx, f.cfg.AcceptTimeout) + defer cancelAccept() + res, err := f.cfg.Acceptor.Accept(acceptCtx, bundleimport.Request{ + Bundle: b, Origin: origin, ExpectedNZB: canonicalIDs(req.NZB), ExpectedNZBSizes: encodedSizes(req.NZB), + Target: bundleimport.Target{Dir: req.TargetDir, Category: req.Category, QueueID: int64(req.QueueID), + TargetPath: req.TargetPath, SkipArr: req.SkipArr}, + }) + if err != nil { + slog.InfoContext(ctx, "Peer bundle not used; importing normally", "reason", string(bundleimport.ReasonOf(err))) + return importer.PeerBundleOutcome{}, false + } + return importer.PeerBundleOutcome{ResultPath: res.Dir, WrittenPaths: res.Paths, TotalSize: res.TotalSize}, true +} + +// MayHaveOffers implements importer.PeerBundleFastPath. +func (f *fastPath) MayHaveOffers(ctx context.Context) bool { + if !f.cfg.Enabled() { + return false + } + return f.cfg.AnyOffers == nil || f.cfg.AnyOffers(ctx) +} + +func (f *fastPath) HoldQueueItem(ctx context.Context, queueID int64) bool { + return f.cfg.Acceptor.QueueHasPending(ctx, queueID) +} + +func (f *fastPath) AttachQueueItem(ctx context.Context, paths []string, queueID int64) error { + return f.cfg.Acceptor.AttachQueue(ctx, paths, queueID) +} + +// ReleaseProber adapts the importer's release probe to bundleimport. +type ReleaseProber interface { + ProbePeerRelease(ctx context.Context, files []importer.PeerProbeList) error +} + +type proberAdapter struct{ p ReleaseProber } + +// NewReleaseProber returns the bundle import's release prober over the +// importer's own probe. +func NewReleaseProber(p ReleaseProber) bundleimport.ReleaseProber { return proberAdapter{p: p} } + +func (a proberAdapter) ProbeRelease(ctx context.Context, files []bundleimport.ProbeFile) error { + lists := make([]importer.PeerProbeList, len(files)) + for i, f := range files { + lists[i] = importer.PeerProbeList{Name: f.Name, SegmentIDs: f.SegmentIDs} + } + err := a.p.ProbePeerRelease(ctx, lists) + switch { + case err == nil: + return nil + case errors.Is(err, importer.ErrPeerProbeMissing): + return bundleimport.ErrProbeMissing + default: + return bundleimport.ErrProbeInconclusive + } +} + +// ImportService is the part of *importer.Service the bundle import needs. +type ImportService interface { + AnnounceVerifiedImport(ctx context.Context, queueID int64, category, targetPath, resultingPath string) error + RecordPeerImport(ctx context.Context, rec importer.PeerImportRecord) (int64, error) + FailImport(ctx context.Context, queueID int64, reason string) error + RemoveFailedFile(ctx context.Context, virtualPath string) error + ReleaseHold(ctx context.Context, queueID int64) error +} + +type serviceAdapter struct{ s ImportService } + +// NewAnnouncer returns the bundle import's announcer over the importer. +func NewAnnouncer(s ImportService) bundleimport.Announcer { return serviceAdapter{s: s} } + +// NewRecorder returns the bundle import's direct-import recorder over the importer. +func NewRecorder(s ImportService) bundleimport.Recorder { return serviceAdapter{s: s} } + +func (a serviceAdapter) AnnounceVerifiedImport(ctx context.Context, an bundleimport.Announcement) error { + return a.s.AnnounceVerifiedImport(ctx, an.QueueID, an.Category, an.TargetPath, an.ResultingPath) +} + +func (a serviceAdapter) FailImport(ctx context.Context, queueID int64, reason string) error { + return a.s.FailImport(ctx, queueID, reason) +} + +func (a serviceAdapter) RemoveFailedFile(ctx context.Context, virtualPath string) error { + return a.s.RemoveFailedFile(ctx, virtualPath) +} + +func (a serviceAdapter) ReleaseHold(ctx context.Context, queueID int64) error { + return a.s.ReleaseHold(ctx, queueID) +} + +func (a serviceAdapter) RecordPeerImport(ctx context.Context, rec bundleimport.ImportRecord) (int64, error) { + return a.s.RecordPeerImport(ctx, importer.PeerImportRecord{ + Title: rec.Title, Category: rec.Category, TargetPath: rec.TargetPath, ResultingPath: rec.ResultingPath, + WrittenPaths: rec.WrittenPaths, TotalSize: rec.TotalSize, ReleaseID: rec.ReleaseID, SkipArr: rec.SkipArr, + Metadata: rec.Metadata, DownloadID: rec.DownloadID, + }) +} + +// BundleService is the part of the peer network service the bundle import +// needs: fetching a verified bundle and learning how its full check ended. +type BundleService interface { + FetchBundle(ctx context.Context, releaseID string) (*bundle.Bundle, service.FetchSource, error) + NoteBundleFullCheck(networkID, publisherID string, passed bool) +} + +type bundleFetcher struct{ s BundleService } + +// NewBundleFetcher adapts the service to bundleimport.Fetcher. A nil service +// yields nil. +func NewBundleFetcher(s BundleService) bundleimport.Fetcher { + if s == nil { + return nil + } + return bundleFetcher{s: s} +} + +func (f bundleFetcher) FetchBundle(ctx context.Context, releaseID string) (*bundle.Bundle, bundleimport.Origin, error) { + b, src, err := f.s.FetchBundle(ctx, releaseID) + if err != nil { + return nil, bundleimport.Origin{}, err + } + return b, bundleimport.Origin{NetworkID: src.Network.String(), PublisherID: src.Publisher.String(), + Open: src.Network.IsOpen(), Corroborators: src.Corroborators}, nil +} + +type bundleReporter struct{ s BundleService } + +// NewBundleReporter adapts the service to bundleimport.BadBundleReporter (and +// BundleVerifiedReporter). A nil service yields nil. +func NewBundleReporter(s BundleService) bundleimport.BadBundleReporter { + if s == nil { + return nil + } + return bundleReporter{s: s} +} + +func (r bundleReporter) ReportBadBundle(_ context.Context, o bundleimport.Origin, _ string) { + r.s.NoteBundleFullCheck(o.NetworkID, o.PublisherID, false) +} + +func (r bundleReporter) ReportVerifiedBundle(_ context.Context, o bundleimport.Origin, _ string) { + r.s.NoteBundleFullCheck(o.NetworkID, o.PublisherID, true) +} diff --git a/internal/peernet/bridge/bundleimport_test.go b/internal/peernet/bridge/bundleimport_test.go new file mode 100644 index 000000000..470082d08 --- /dev/null +++ b/internal/peernet/bridge/bundleimport_test.go @@ -0,0 +1,236 @@ +package bridge + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/importer" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/bundleimport" + "github.com/kipsilabs/altmount/internal/peernet/contentid" + "github.com/kipsilabs/altmount/internal/peernet/identity" + "github.com/kipsilabs/altmount/internal/peernet/netid" + "github.com/kipsilabs/altmount/internal/peernet/service" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeAcceptor struct { + reqs []bundleimport.Request + res *bundleimport.Result + err error +} + +func (a *fakeAcceptor) Accept(_ context.Context, r bundleimport.Request) (*bundleimport.Result, error) { + a.reqs = append(a.reqs, r) + return a.res, a.err +} +func (a *fakeAcceptor) QueueHasPending(context.Context, int64) bool { return true } +func (a *fakeAcceptor) AttachQueue(context.Context, []string, int64) error { return nil } + +type blockingFetcher struct { + b *bundle.Bundle + err error + calls atomic.Int32 + ids []string + block bool +} + +func (f *blockingFetcher) FetchBundle(ctx context.Context, id string) (*bundle.Bundle, bundleimport.Origin, error) { + f.calls.Add(1) + f.ids = append(f.ids, id) + if f.block { + <-ctx.Done() + return nil, bundleimport.Origin{}, ctx.Err() + } + return f.b, bundleimport.Origin{NetworkID: "n", PublisherID: "p"}, f.err +} + +func nzbStore() *metapb.NzbStore { + return &metapb.NzbStore{Files: []*metapb.NzbFileEntry{ + {Segments: []*metapb.NzbSeg{{Id: "a@x"}, {Id: ""}}}, + {Segments: []*metapb.NzbSeg{{Id: "c@x"}}}, + }} +} + +func hook(t *testing.T, a *fakeAcceptor, f *blockingFetcher, enabled bool, timeout time.Duration) importer.PeerBundleFastPath { + t.Helper() + h := NewPeerBundleFastPath(FastPathConfig{Acceptor: a, Fetcher: f, Enabled: func() bool { return enabled }, FetchTimeout: timeout}) + require.NotNil(t, h) + return h +} + +func matching(t *testing.T) *bundle.Bundle { + rid, err := contentid.ReleaseID(nzbStore()) + require.NoError(t, err) + return &bundle.Bundle{Scheme: bundle.SchemeNZB, ReleaseID: rid} +} + +func req() importer.PeerBundleRequest { + return importer.PeerBundleRequest{NZB: nzbStore(), QueueID: 3, TargetDir: "movies/R", Category: "movies", SkipArr: true} +} + +func TestFastPath_DisabledDoesNothing(t *testing.T) { + f := &blockingFetcher{b: matching(t)} + _, ok := hook(t, &fakeAcceptor{}, f, false, time.Second).TryBundle(context.Background(), req()) + assert.False(t, ok) + assert.Zero(t, f.calls.Load()) +} + +func TestFastPath_NilCollaboratorsYieldNoHook(t *testing.T) { + assert.Nil(t, NewPeerBundleFastPath(FastPathConfig{Enabled: func() bool { return true }})) +} + +func TestFastPath_TakenAsksForTheExactReleaseAndCrossChecksTheNZB(t *testing.T) { + a := &fakeAcceptor{res: &bundleimport.Result{Dir: "movies/R", Paths: []string{"movies/R/a.mkv"}, TotalSize: 5}} + f := &blockingFetcher{b: matching(t)} + out, ok := hook(t, a, f, true, time.Second).TryBundle(context.Background(), req()) + require.True(t, ok) + assert.Equal(t, importer.PeerBundleOutcome{ResultPath: "movies/R", WrittenPaths: []string{"movies/R/a.mkv"}, TotalSize: 5}, out) + rid, _ := contentid.ReleaseID(nzbStore()) + assert.Equal(t, []string{rid.String()}, f.ids) + require.Len(t, a.reqs, 1) + assert.Equal(t, [][]string{{"a@x", "b@x"}, {"c@x"}}, a.reqs[0].ExpectedNZB, "the user's own list, canonical") + assert.Equal(t, bundleimport.Target{Dir: "movies/R", Category: "movies", QueueID: 3, SkipArr: true}, a.reqs[0].Target) +} + +func TestFastPath_FailsOpen(t *testing.T) { + other := matching(t) + other.ReleaseID[0] ^= 1 + files := matching(t) + files.Scheme = bundle.SchemeFiles + for name, tc := range map[string]struct { + f *blockingFetcher + a *fakeAcceptor + }{ + "fetch error": {&blockingFetcher{err: errors.New("no peers")}, &fakeAcceptor{}}, + "nothing found": {&blockingFetcher{}, &fakeAcceptor{}}, + "other release": {&blockingFetcher{b: other}, &fakeAcceptor{}}, + "files scheme": {&blockingFetcher{b: files}, &fakeAcceptor{}}, + "accept rejection": {&blockingFetcher{b: matching(t)}, &fakeAcceptor{err: errors.New("rejected")}}, + } { + t.Run(name, func(t *testing.T) { + _, ok := hook(t, tc.a, tc.f, true, time.Second).TryBundle(context.Background(), req()) + assert.False(t, ok) + }) + } +} + +func TestFastPath_SlowFetchIsCutOffAtTheDeadline(t *testing.T) { + f := &blockingFetcher{block: true} + start := time.Now() + _, ok := hook(t, &fakeAcceptor{}, f, true, 50*time.Millisecond).TryBundle(context.Background(), req()) + assert.False(t, ok) + assert.Less(t, time.Since(start), 2*time.Second, "never slower than the deadline") +} + +func TestReleaseProberMapsErrors(t *testing.T) { + for in, want := range map[error]error{ + nil: nil, + importer.ErrPeerProbeMissing: bundleimport.ErrProbeMissing, + importer.ErrPeerProbeInconclusive: bundleimport.ErrProbeInconclusive, + errors.New("anything else"): bundleimport.ErrProbeInconclusive, + } { + err := NewReleaseProber(probeFn(func() error { return in })).ProbeRelease(context.Background(), []bundleimport.ProbeFile{{Name: "a", SegmentIDs: []string{"x"}}}) + assert.Equal(t, want, err) + } + _ = strings.Repeat +} + +type probeFn func() error + +func (p probeFn) ProbePeerRelease(context.Context, []importer.PeerProbeList) error { return p() } + +type fakeBundleService struct { + b *bundle.Bundle + src service.FetchSource + err error + noted []string + gotRel string +} + +func (f *fakeBundleService) FetchBundle(_ context.Context, rel string) (*bundle.Bundle, service.FetchSource, error) { + f.gotRel = rel + return f.b, f.src, f.err +} +func (f *fakeBundleService) NoteBundleFullCheck(n, p string, passed bool) { + f.noted = append(f.noted, n+"/"+p+"/"+map[bool]string{true: "pass", false: "fail"}[passed]) +} + +func TestBundleFetcherAdapterMapsSourceToOrigin(t *testing.T) { + nw, pub := netid.Open(), identity.NodeID{1} + svc := &fakeBundleService{b: &bundle.Bundle{}, src: service.FetchSource{Network: nw, Publisher: pub}} + b, o, err := NewBundleFetcher(svc).FetchBundle(context.Background(), "rel") + require.NoError(t, err) + assert.Same(t, svc.b, b) + assert.Equal(t, bundleimport.Origin{NetworkID: nw.String(), PublisherID: pub.String(), Open: true}, o) + assert.Equal(t, "rel", svc.gotRel) + svc.err = errors.New("no offers") + _, _, err = NewBundleFetcher(svc).FetchBundle(context.Background(), "rel") + assert.Error(t, err) + assert.Nil(t, NewBundleFetcher(nil)) +} + +func TestBundleReporterAdapterPassesFailureAndSuccess(t *testing.T) { + svc := &fakeBundleService{} + rep := NewBundleReporter(svc) + o := bundleimport.Origin{NetworkID: "n", PublisherID: "p"} + rep.ReportBadBundle(context.Background(), o, "rel") + rep.(bundleimport.BundleVerifiedReporter).ReportVerifiedBundle(context.Background(), o, "rel") + assert.Equal(t, []string{"n/p/fail", "n/p/pass"}, svc.noted) + assert.Nil(t, NewBundleReporter(nil)) +} + +type originFetcher struct { + b *bundle.Bundle + o bundleimport.Origin +} + +func (f originFetcher) FetchBundle(context.Context, string) (*bundle.Bundle, bundleimport.Origin, error) { + return f.b, f.o, nil +} + +func TestFastPath_OpenNetworkNeedsTheSwitchAndTwoCorroboratingPublishers(t *testing.T) { + cases := []struct { + name string + origin bundleimport.Origin + on bool + taken bool + }{ + {"private network, switch off", bundleimport.Origin{NetworkID: "p", Open: false}, false, true}, + {"open network, switch off", bundleimport.Origin{NetworkID: "o", Open: true, Corroborators: 5}, false, false}, + {"open network, switch on, one publisher", bundleimport.Origin{NetworkID: "o", Open: true, Corroborators: 1}, true, false}, + {"open network, switch on, two publishers", bundleimport.Origin{NetworkID: "o", Open: true, Corroborators: 2}, true, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + a := &fakeAcceptor{res: &bundleimport.Result{Dir: "movies/R", Paths: []string{"movies/R/a.mkv"}}} + h := NewPeerBundleFastPath(FastPathConfig{Acceptor: a, Fetcher: originFetcher{b: matching(t), o: c.origin}, + Enabled: func() bool { return true }, OpenFastPath: func() bool { return c.on }}) + _, ok := h.TryBundle(context.Background(), req()) + assert.Equal(t, c.taken, ok) + assert.Equal(t, c.taken, len(a.reqs) == 1, "an open offer that fails the policy is never even accepted") + }) + } +} + +type slowAcceptor struct{ fakeAcceptor } + +func (a *slowAcceptor) Accept(ctx context.Context, r bundleimport.Request) (*bundleimport.Result, error) { + <-ctx.Done() + return nil, ctx.Err() +} + +func TestFastPath_VerificationThatDoesNotFitDeclinesInsteadOfStallingTheImport(t *testing.T) { + h := NewPeerBundleFastPath(FastPathConfig{Acceptor: &slowAcceptor{}, Fetcher: originFetcher{b: matching(t), o: bundleimport.Origin{NetworkID: "p"}}, + Enabled: func() bool { return true }, AcceptTimeout: 60 * time.Millisecond}) + start := time.Now() + _, ok := h.TryBundle(context.Background(), req()) + assert.False(t, ok) + assert.Less(t, time.Since(start), 2*time.Second) +} diff --git a/internal/peernet/bridge/nooffer_bench_test.go b/internal/peernet/bridge/nooffer_bench_test.go new file mode 100644 index 000000000..27a4a35bd --- /dev/null +++ b/internal/peernet/bridge/nooffer_bench_test.go @@ -0,0 +1,51 @@ +package bridge + +import ( + "context" + "fmt" + "sync/atomic" + "testing" + + "github.com/javi11/nzbparser" + "github.com/kipsilabs/altmount/internal/metadata" + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +// syntheticNzb is an NZB of files*perFile segments. +func syntheticNzb(files, perFile int) *nzbparser.Nzb { + n := &nzbparser.Nzb{} + for f := 0; f < files; f++ { + nf := nzbparser.NzbFile{Subject: fmt.Sprintf("release [%d/%d] \"part%03d.rar\" yEnc", f+1, files, f), Poster: "p@x", Groups: []string{"alt.binaries.test"}} + for s := 1; s <= perFile; s++ { + nf.Segments = append(nf.Segments, nzbparser.NzbSegment{Number: s, Bytes: 750_000, ID: fmt.Sprintf("f%03d-s%05d@example.invalid", f, s)}) + } + n.Files = append(n.Files, nf) + } + return n +} + +// What every import paid while the hook was installed and no peer offered +// anything: the NZB store plus the Release ID (before), against one cached +// answer (after). 100,000 segments. +func BenchmarkNoOfferImport_Before(b *testing.B) { + n := syntheticNzb(100, 1000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + store, _ := metadata.BuildStore(n) + if _, err := contentid.ReleaseID(store); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkNoOfferImport_After(b *testing.B) { + var gen atomic.Uint64 + check := NewOfferCache(&fakeOfferSource{}, gen.Load) + ctx := context.Background() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if check(ctx) { + b.Fatal("no offer expected") + } + } +} diff --git a/internal/peernet/bridge/offercache.go b/internal/peernet/bridge/offercache.go new file mode 100644 index 000000000..cc5c3372b --- /dev/null +++ b/internal/peernet/bridge/offercache.go @@ -0,0 +1,39 @@ +package bridge + +import ( + "context" + "sync" +) + +// OfferSource answers whether any stored record announces a bundle. +type OfferSource interface { + AnyBundleOffer(ctx context.Context) (bool, error) +} + +// NewOfferCache returns a cheap "does this node hold any bundle offer" check for +// FastPathConfig.AnyOffers. The answer is cached and recomputed only when +// generation (a counter of bundle-announcing records stored) changed, so with +// no offer around every import pays one atomic load. A lookup error is never +// cached and answers true: an unsure answer must not hide an offer. +func NewOfferCache(src OfferSource, generation func() uint64) func(ctx context.Context) bool { + var ( + mu sync.Mutex + valid bool + cached bool + seen uint64 + ) + return func(ctx context.Context) bool { + mu.Lock() + defer mu.Unlock() + gen := generation() + if valid && gen == seen { + return cached + } + any, err := src.AnyBundleOffer(ctx) + if err != nil { + return true + } + valid, cached, seen = true, any, gen + return any + } +} diff --git a/internal/peernet/bridge/offercache_test.go b/internal/peernet/bridge/offercache_test.go new file mode 100644 index 000000000..33dd1c086 --- /dev/null +++ b/internal/peernet/bridge/offercache_test.go @@ -0,0 +1,48 @@ +package bridge + +import ( + "context" + "errors" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" +) + +type fakeOfferSource struct { + any atomic.Bool + err error + calls atomic.Int32 +} + +func (f *fakeOfferSource) AnyBundleOffer(context.Context) (bool, error) { + f.calls.Add(1) + return f.any.Load(), f.err +} + +func TestOfferCache_AsksOnceUntilARecordArrives(t *testing.T) { + var gen atomic.Uint64 + src := &fakeOfferSource{} + check := NewOfferCache(src, gen.Load) + ctx := context.Background() + + for i := 0; i < 100; i++ { + assert.False(t, check(ctx)) + } + assert.Equal(t, int32(1), src.calls.Load(), "no offer: one lookup, then cached") + + src.any.Store(true) + assert.False(t, check(ctx), "no record arrived: still the cached answer") + gen.Add(1) // a bundle-announcing record arrived + assert.True(t, check(ctx), "the arrival invalidates the cache") + assert.True(t, check(ctx)) + assert.Equal(t, int32(2), src.calls.Load()) +} + +func TestOfferCache_ErrorIsNeverCachedAndSaysYes(t *testing.T) { + src := &fakeOfferSource{err: errors.New("db busy")} + check := NewOfferCache(src, func() uint64 { return 0 }) + assert.True(t, check(context.Background())) + src.err = nil + assert.False(t, check(context.Background()), "the next call looks again") +} diff --git a/internal/peernet/bridge/passwordorigin.go b/internal/peernet/bridge/passwordorigin.go new file mode 100644 index 000000000..16dc12a6a --- /dev/null +++ b/internal/peernet/bridge/passwordorigin.go @@ -0,0 +1,75 @@ +package bridge + +import ( + "context" + "log/slog" + + "github.com/kipsilabs/altmount/internal/importer" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +// MetadataReader reads resolved file metadata (nil, nil when absent). +type MetadataReader interface { + ReadFileMetadata(virtualPath string) (*metapb.FileMetadata, error) +} + +// PasswordOriginStore persists which files' passwords came with their NZB. +type PasswordOriginStore interface { + MarkFromNZB(ctx context.Context, fileIDs []string) error +} + +type pwRecorder struct { + meta MetadataReader + store PasswordOriginStore +} + +// NewNZBPasswordRecorder returns the importer's hook that records which +// written files carry key material derived from a password that came with the +// NZB's own content. Only files that carry an AES key (their own or an outer +// volume's) are recorded, by File ID, so the record survives renames. It never +// sees the password and never fails an import. +func NewNZBPasswordRecorder(meta MetadataReader, store PasswordOriginStore) importer.NZBPasswordRecorder { + if meta == nil || store == nil { + return nil + } + return pwRecorder{meta: meta, store: store} +} + +func hasKeyMaterial(m *metapb.FileMetadata) bool { + if len(m.GetAesKey()) > 0 { + return true + } + for _, o := range m.GetSharedOuterSources() { + if len(o.GetAesKey()) > 0 { + return true + } + } + for _, o := range m.GetNestedSources() { + if len(o.GetAesKey()) > 0 { + return true + } + } + return false +} + +func (r pwRecorder) RecordNZBPasswordFiles(ctx context.Context, paths []string) { + ids := make([]string, 0, len(paths)) + for _, p := range paths { + m, err := r.meta.ReadFileMetadata(p) + if err != nil || m == nil || !hasKeyMaterial(m) { + continue + } + id, err := contentid.FileID(m) + if err != nil { + continue + } + ids = append(ids, id.String()) + } + if len(ids) == 0 { + return + } + if err := r.store.MarkFromNZB(ctx, ids); err != nil { + slog.WarnContext(ctx, "Could not record where an archive password came from", "files", len(ids), "error", err) + } +} diff --git a/internal/peernet/bridge/passwordorigin_test.go b/internal/peernet/bridge/passwordorigin_test.go new file mode 100644 index 000000000..7c6e5a1ce --- /dev/null +++ b/internal/peernet/bridge/passwordorigin_test.go @@ -0,0 +1,57 @@ +package bridge + +import ( + "context" + "errors" + "testing" + + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/contentid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeMeta map[string]*metapb.FileMetadata + +func (f fakeMeta) ReadFileMetadata(p string) (*metapb.FileMetadata, error) { return f[p], nil } + +type fakeOriginStore struct { + ids []string + err error +} + +func (f *fakeOriginStore) MarkFromNZB(_ context.Context, ids []string) error { + f.ids = append(f.ids, ids...) + return f.err +} + +func seg(id string) []*metapb.SegmentData { + return []*metapb.SegmentData{{Id: id, SegmentSize: 100, EndOffset: 99}} +} + +func fileID(t *testing.T, m *metapb.FileMetadata) string { + t.Helper() + id, err := contentid.FileID(m) + require.NoError(t, err) + return id.String() +} + +func TestNZBPasswordRecorder_OnlyKeyedFilesAreRecordedByFileID(t *testing.T) { + keyed := &metapb.FileMetadata{FileSize: 100, SegmentData: seg("k@x"), Encryption: metapb.Encryption_AES, + AesKey: make([]byte, 32), AesIv: make([]byte, 16)} + nested := &metapb.FileMetadata{FileSize: 100, SegmentData: seg("n@x"), + SharedOuterSources: []*metapb.NestedSegmentSource{{Segments: seg("o@x"), AesKey: make([]byte, 32), AesIv: make([]byte, 16), InnerLength: 100}}} + plain := &metapb.FileMetadata{FileSize: 100, SegmentData: seg("p@x")} + store := &fakeOriginStore{} + NewNZBPasswordRecorder(fakeMeta{"a/k.mkv": keyed, "a/n.mkv": nested, "a/p.mkv": plain}, store). + RecordNZBPasswordFiles(context.Background(), []string{"a/k.mkv", "a/n.mkv", "a/p.mkv", "a/gone.mkv"}) + assert.ElementsMatch(t, []string{fileID(t, keyed), fileID(t, nested)}, store.ids, + "keyed files only; unkeyed and missing files have no key to protect") +} + +func TestNZBPasswordRecorder_StoreFailureNeverPanicsOrLeaks(t *testing.T) { + keyed := &metapb.FileMetadata{FileSize: 100, SegmentData: seg("k@x"), AesKey: make([]byte, 32), AesIv: make([]byte, 16)} + store := &fakeOriginStore{err: errors.New("db down")} + NewNZBPasswordRecorder(fakeMeta{"k": keyed}, store).RecordNZBPasswordFiles(context.Background(), []string{"k"}) + assert.Len(t, store.ids, 1) +} diff --git a/internal/peernet/bundle/budget_test.go b/internal/peernet/bundle/budget_test.go new file mode 100644 index 000000000..e08c74241 --- /dev/null +++ b/internal/peernet/bundle/budget_test.go @@ -0,0 +1,74 @@ +package bundle + +import ( + "runtime" + "testing" + + "google.golang.org/protobuf/encoding/protowire" +) + +// hostileClipBundle is the shape from the review: many tiny files, each with +// the maximum number of clip pairs. It is a few tens of KB on the wire and +// about 62 MB canonical. +func hostileClipBundle(t testing.TB, files int) []byte { + t.Helper() + var clips []byte + for i := 0; i < MaxClips; i++ { + clips = protowire.AppendVarint(clips, 192) + clips = protowire.AppendVarint(clips, 0) + } + var out []byte + out = appendVarint(out, fVersion, SchemaVersion) + out = appendVarint(out, fScheme, uint64(SchemeFiles)) + out = appendBytes(out, fReleaseID, repeat32(0xAA)) + out = appendString(out, fTitle, "t") + out = appendVarint(out, fPosted, 1700000000) + out = appendVarint(out, fTotal, uint64(files)) + for i := 0; i < files; i++ { + var f []byte + f = appendString(f, fName, string(rune('a'+i%26))+string(rune('a'+i/26%26))+string(rune('a'+i/676))+".bin") + f = appendVarint(f, fSize, 1) + f = appendBytes(f, fClips, clips) + id := repeat32(0) + id[0], id[1] = byte(i>>8), byte(i) + id[31] = 1 + f = appendBytes(f, fFileID, id) + out = appendBytes(out, fFiles, f) + } + return out +} + +func repeat32(v byte) []byte { + b := make([]byte, 32) + for i := range b { + b[i] = v + } + return b +} + +func TestHostileClipsCannotBloatVerify(t *testing.T) { + raw := hostileClipBundle(t, 200) + if len(raw) > MaxDecompressedSize { + t.Fatalf("test input is %d bytes canonical", len(raw)) + } + wire, err := Compress(raw) + if err != nil { + t.Fatal(err) + } + if len(wire) > 64<<10 { + t.Fatalf("hostile wire is %d bytes, want at most 64 KB", len(wire)) + } + id := ComputeID(raw) + + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + _, err = VerifyCanonical(raw, id, [32]byte{0xAA}, Options{}) + runtime.ReadMemStats(&after) + if err == nil { + t.Fatal("hostile bundle verified") + } + if got := after.TotalAlloc - before.TotalAlloc; got > 128<<20 { + t.Fatalf("Verify allocated %d MB for a %d byte input, want at most 128 MB", got>>20, len(wire)) + } +} diff --git a/internal/peernet/bundle/build.go b/internal/peernet/bundle/build.go new file mode 100644 index 000000000..12a16b9c9 --- /dev/null +++ b/internal/peernet/bundle/build.go @@ -0,0 +1,425 @@ +package bundle + +import ( + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/kipsilabs/altmount/internal/holes" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +// FileInput is one local file of a release handed to the builder. +type FileInput struct { + // Name is the original file name; when empty Meta.OriginalFilename is + // used. It is sanitised to a base name. + Name string + // Meta is the file's metadata with its segments resolved (inline + // SegmentData, nested sources with segments or shared outer sources), as + // MetadataService.ReadFileMetadata returns it. It is not modified. + Meta *metapb.FileMetadata + // PasswordFromNZB says that any encryption key in Meta derives from a + // password that is already in the NZB. The publisher alone can know this. + // It only matters for a file that carries key material: when it is false + // such a file is not shareable and neither is the release. + PasswordFromNZB bool + // PatchedLocally marks a file that was healed by local PAR2 patches. Its + // articles do not match what the network would fetch, so it is not + // shareable. + PatchedLocally bool +} + +// ReleaseInput is everything the builder needs. It is plain data: no I/O. +type ReleaseInput struct { + Scheme Scheme + Title string + PostedAt time.Time + // NZBPassword is the password found in the NZB, if any. It travels as an + // opaque string and is never used by the receiver at import. + NZBPassword string + // NZBFiles (SchemeNZB only) lists the message-IDs of every NZB file in NZB + // order, including files that were not imported. Gap placeholders are + // skipped. + NZBFiles [][]string + Files []FileInput +} + +// ShareReason says why a release cannot be shared. +type ShareReason string + +const ( + ReasonRclone ShareReason = "rclone_encrypted" + ReasonOtherEncryption ShareReason = "unsupported_encryption" + ReasonPasswordNotNZB ShareReason = "password_not_from_nzb" + ReasonLocalPatch ShareReason = "healed_by_local_patch" +) + +// NotShareableError reports the first file that makes the release not +// shareable. errors.Is(err, ErrNotShareable) is true for it. +type NotShareableError struct { + File string + Reason ShareReason +} + +func (e *NotShareableError) Error() string { + return fmt.Sprintf("%v: file %q: %s", ErrNotShareable, e.File, e.Reason) +} + +// Is makes errors.Is(err, ErrNotShareable) match. +func (e *NotShareableError) Is(target error) bool { return target == ErrNotShareable } + +func hasKeyMaterial(m *metapb.FileMetadata) bool { + if len(m.AesKey) > 0 { + return true + } + for _, ns := range m.NestedSources { + if len(ns.AesKey) > 0 { + return true + } + } + for _, ns := range m.SharedOuterSources { + if len(ns.AesKey) > 0 { + return true + } + } + return false +} + +func checkShareable(fi *FileInput, name string) error { + m := fi.Meta + switch m.Encryption { + case metapb.Encryption_NONE, metapb.Encryption_AES: + case metapb.Encryption_RCLONE: + return &NotShareableError{File: name, Reason: ReasonRclone} + default: + return &NotShareableError{File: name, Reason: ReasonOtherEncryption} + } + switch { + case fi.PatchedLocally: + return &NotShareableError{File: name, Reason: ReasonLocalPatch} + case hasKeyMaterial(m) && !fi.PasswordFromNZB: + return &NotShareableError{File: name, Reason: ReasonPasswordNotNZB} + } + return nil +} + +type preparedFile struct { + in *FileInput + name string + id contentid.ID +} + +func mapIDError(err error) error { + switch { + case errors.Is(err, contentid.ErrEmptyMessageID): + return fmt.Errorf("%w: %v", ErrBadMessageID, err) + case errors.Is(err, contentid.ErrUnresolved): + return fmt.Errorf("%w: %v", ErrUnresolved, err) + case errors.Is(err, contentid.ErrBadSharedSource): + return fmt.Errorf("%w: %v", ErrOutOfRange, err) + default: + return fmt.Errorf("%w: %v", ErrInvalidField, err) + } +} + +// splitExt returns name without a short trailing extension, and the extension. +func splitExt(name string) (string, string) { + if i := strings.LastIndexByte(name, '.'); i > 0 && len(name)-i <= maxExtLen { + return name[:i], name[i:] + } + return name, "" +} + +// uniqueName returns name, or "stem (n).ext" for the smallest n >= 2 that is +// free, and records it in taken. Comparison ignores case. +func uniqueName(name string, taken map[string]struct{}) string { + cand := name + stem, ext := splitExt(name) + for n := 2; ; n++ { + if _, dup := taken[strings.ToLower(cand)]; !dup { + taken[strings.ToLower(cand)] = struct{}{} + return cand + } + cand = SanitizeFilename(fmt.Sprintf("%s (%d)%s", stem, n, ext)) + } +} + +// poolBuilder maps message-IDs to pool indices. +type poolBuilder struct { + nzb bool + idx map[string]int64 + pool []string +} + +func (p *poolBuilder) ref(raw string) (int64, error) { + if holes.IsPlaceholderID(raw) { + return 0, fmt.Errorf("%w: file has a known hole", ErrNotShareable) + } + id, err := CanonicalMessageID(raw) + if err != nil { + return 0, err + } + if i, ok := p.idx[id]; ok { + return i, nil + } + if p.nzb { + return 0, fmt.Errorf("%w: %q", ErrSegmentNotInNZB, id) + } + i := int64(len(p.pool)) + p.idx[id] = i + p.pool = append(p.pool, id) + return i, nil +} + +func (p *poolBuilder) table(in []*metapb.SegmentData) ([]Segment, error) { + if len(in) == 0 { + return nil, nil + } + out := make([]Segment, len(in)) + for i, s := range in { + if s == nil { + return nil, fmt.Errorf("%w: nil segment", ErrInvalidField) + } + r, err := p.ref(s.Id) + if err != nil { + return nil, err + } + out[i] = Segment{Ref: r, Size: s.SegmentSize, Start: s.StartOffset, End: s.EndOffset} + } + return out, nil +} + +// outerKey identifies an outer source by content, to share identical ones. +func outerKey(ns *metapb.NestedSegmentSource) string { + h := sha256.New() + var b [8]byte + w := func(v int64) { + binary.BigEndian.PutUint64(b[:], uint64(v)) + h.Write(b[:]) + } + w(int64(len(ns.Segments))) + for _, s := range ns.Segments { + w(int64(len(s.Id))) + h.Write([]byte(s.Id)) + w(s.SegmentSize) + w(s.StartOffset) + w(s.EndOffset) + } + w(int64(len(ns.AesKey))) + h.Write(ns.AesKey) + w(int64(len(ns.AesIv))) + h.Write(ns.AesIv) + w(ns.InnerVolumeSize) + return string(h.Sum(nil)) +} + +// effectiveOuter returns the outer source a nested source reads through, +// following a shared reference. Indices were checked by contentid.FileID. +func effectiveOuter(m *metapb.FileMetadata, ns *metapb.NestedSegmentSource) *metapb.NestedSegmentSource { + if ns.SharedOuterSourceIndex > 0 { + sh := m.SharedOuterSources[ns.SharedOuterSourceIndex-1] + eff := &metapb.NestedSegmentSource{Segments: sh.Segments, AesKey: sh.AesKey, AesIv: sh.AesIv, InnerVolumeSize: sh.InnerVolumeSize} + if ns.InnerVolumeSize != 0 { + eff.InnerVolumeSize = ns.InnerVolumeSize + } + return eff + } + return ns +} + +func (p *poolBuilder) file(pf preparedFile) (File, error) { + m := pf.in.Meta + if (len(m.AesKey) > 0) != (m.Encryption == metapb.Encryption_AES) { + return File{}, fmt.Errorf("%w: encryption type and key disagree", ErrBadEncryption) + } + f := File{Name: pf.name, Size: m.FileSize, ReleaseDate: m.ReleaseDate, FileID: pf.id, + AESKey: append([]byte(nil), m.AesKey...), AESIV: append([]byte(nil), m.AesIv...)} + if len(f.AESKey) == 0 { + f.AESKey, f.AESIV = nil, nil + } + var err error + if f.Segments, err = p.table(m.SegmentData); err != nil { + return File{}, err + } + index := map[string]int{} + for _, ns := range m.NestedSources { + if ns == nil { + return File{}, fmt.Errorf("%w: nil nested source", ErrInvalidField) + } + eff := effectiveOuter(m, ns) + k := outerKey(eff) + oi, ok := index[k] + if !ok { + segs, err := p.table(eff.Segments) + if err != nil { + return File{}, err + } + o := OuterSource{Segments: segs, VolumeSize: eff.InnerVolumeSize} + if len(eff.AesKey) > 0 { + o.AESKey, o.AESIV = append([]byte(nil), eff.AesKey...), append([]byte(nil), eff.AesIv...) + } + oi = len(f.Outer) + index[k] = oi + f.Outer = append(f.Outer, o) + } + f.Extents = append(f.Extents, Extent{Outer: oi, InnerOffset: ns.InnerOffset, InnerLength: ns.InnerLength}) + } + for _, c := range m.ClipBoundaries { + if c == nil { + return File{}, fmt.Errorf("%w: nil clip", ErrBadClip) + } + f.Clips = append(f.Clips, Clip{ByteLen: c.ByteLen, Delta90k: c.Delta_90K}) + } + taken := map[string]struct{}{} + for _, pr := range m.Par2Files { + if pr == nil { + return File{}, fmt.Errorf("%w: nil par2 reference", ErrInvalidField) + } + segs, err := p.table(pr.SegmentData) + if err != nil { + return File{}, err + } + f.Par2 = append(f.Par2, Par2File{Name: uniqueName(SanitizeFilename(pr.Filename), taken), Size: pr.FileSize, Segments: segs}) + } + return f, nil +} + +func nzbPool(files [][]string) (*poolBuilder, []int, error) { + if len(files) == 0 || len(files) > MaxReleaseFiles { + if len(files) == 0 { + return nil, nil, fmt.Errorf("%w: the nzb scheme needs the release segment list", ErrBadPool) + } + return nil, nil, fmt.Errorf("%w: %d release files", ErrLimit, len(files)) + } + p := &poolBuilder{nzb: true, idx: map[string]int64{}} + counts := make([]int, len(files)) + for i, f := range files { + for _, raw := range f { + if holes.IsPlaceholderID(raw) { + continue + } + id, err := CanonicalMessageID(raw) + if err != nil { + return nil, nil, err + } + if len(p.pool) >= MaxSegments { + return nil, nil, fmt.Errorf("%w: release segments", ErrLimit) + } + if _, ok := p.idx[id]; !ok { + p.idx[id] = int64(len(p.pool)) + } + p.pool = append(p.pool, id) + counts[i]++ + } + } + return p, counts, nil +} + +// releaseIDOfList computes the nzb-scheme Release ID from a pool and counts. +func releaseIDOfList(pool []string, counts []int) (contentid.ID, error) { + store := &metapb.NzbStore{} + pos := 0 + for _, n := range counts { + fe := &metapb.NzbFileEntry{} + for _, id := range pool[pos : pos+n] { + fe.Segments = append(fe.Segments, &metapb.NzbSeg{Id: id}) + } + pos += n + store.Files = append(store.Files, fe) + } + return contentid.ReleaseID(store) +} + +// FromLocal builds a bundle from a finished local import. It returns a +// *NotShareableError (errors.Is ErrNotShareable) when any file cannot be +// shared, and a typed validation error when the local metadata is itself +// inconsistent. Files are ordered by File ID and named deterministically, so +// the result does not depend on the order of in.Files. Nothing in the input +// is modified or retained. +func FromLocal(in ReleaseInput) (*Bundle, error) { + if in.Scheme != SchemeNZB && in.Scheme != SchemeFiles { + return nil, fmt.Errorf("%w: %d", ErrUnknownScheme, in.Scheme) + } + title := SanitizeTitle(in.Title) + if title == "" { + return nil, fmt.Errorf("%w: title is empty after sanitising", ErrInvalidField) + } + if in.PostedAt.Unix() < 1 { + return nil, fmt.Errorf("%w: no post date", ErrInvalidField) + } + switch { + case len(in.Files) == 0: + return nil, fmt.Errorf("%w: no files", ErrInvalidField) + case len(in.Files) > MaxFiles: + return nil, fmt.Errorf("%w: %d files", ErrLimit, len(in.Files)) + } + prepared := make([]preparedFile, len(in.Files)) + for i := range in.Files { + fi := &in.Files[i] + if fi.Meta == nil { + return nil, fmt.Errorf("%w: file %d has no metadata", ErrInvalidField, i) + } + name := fi.Name + if name == "" { + name = fi.Meta.OriginalFilename + } + name = SanitizeFilename(name) + if err := checkShareable(fi, name); err != nil { + return nil, err + } + id, err := contentid.FileID(fi.Meta) + if err != nil { + return nil, fmt.Errorf("file %q: %w", name, mapIDError(err)) + } + prepared[i] = preparedFile{in: fi, name: name, id: id} + } + sort.SliceStable(prepared, func(a, b int) bool { return string(prepared[a].id[:]) < string(prepared[b].id[:]) }) + taken := map[string]struct{}{} + for i := range prepared { + if i > 0 && prepared[i].id == prepared[i-1].id { + return nil, fmt.Errorf("%w: %s", ErrDuplicateFile, prepared[i].id) + } + prepared[i].name = uniqueName(prepared[i].name, taken) + } + + b := &Bundle{Scheme: in.Scheme, Title: title, PostedUnix: in.PostedAt.Unix(), NZBPassword: in.NZBPassword} + p := &poolBuilder{idx: map[string]int64{}} + if in.Scheme == SchemeNZB { + var counts []int + var err error + if p, counts, err = nzbPool(in.NZBFiles); err != nil { + return nil, err + } + b.ReleaseFiles = counts + } + ids := make([]contentid.ID, len(prepared)) + for i, pf := range prepared { + f, err := p.file(pf) + if err != nil { + return nil, fmt.Errorf("file %q: %w", pf.name, err) + } + b.Files = append(b.Files, f) + ids[i] = pf.id + b.TotalSize += f.Size + } + b.Pool = p.pool + switch in.Scheme { + case SchemeNZB: + rid, err := releaseIDOfList(b.Pool, b.ReleaseFiles) + if err != nil { + return nil, mapIDError(err) + } + b.ReleaseID = rid + default: + b.ReleaseID = FilesReleaseID(ids) + } + if err := Validate(b); err != nil { + return nil, err + } + return b, nil +} diff --git a/internal/peernet/bundle/build_test.go b/internal/peernet/bundle/build_test.go new file mode 100644 index 000000000..8014c03b1 --- /dev/null +++ b/internal/peernet/bundle/build_test.go @@ -0,0 +1,302 @@ +package bundle + +import ( + "bytes" + "errors" + "strings" + "testing" + "time" + + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "google.golang.org/protobuf/proto" +) + +func releaseInput(specs []specFile) ReleaseInput { + in := ReleaseInput{Scheme: SchemeFiles, Title: "Some Release 2024", PostedAt: time.Unix(1700000000, 0)} + for _, s := range specs { + in.Files = append(in.Files, s.input()) + } + return in +} + +func TestFromLocalMatchesIndependentAssembly(t *testing.T) { + got, err := FromLocal(releaseInput(testSpecs())) + if err != nil { + t.Fatalf("FromLocal: %v", err) + } + if !bytes.Equal(mustEncode(t, got), mustEncode(t, goodBundle(t))) { + t.Fatal("files scheme: FromLocal differs from the independently assembled bundle") + } + + in := releaseInput(testSpecs()) + in.Scheme, in.NZBFiles = SchemeNZB, nzbLayout() + got, err = FromLocal(in) + if err != nil { + t.Fatalf("FromLocal nzb: %v", err) + } + if !bytes.Equal(mustEncode(t, got), mustEncode(t, goodNZBBundle(t))) { + t.Fatal("nzb scheme: FromLocal differs from the independently assembled bundle") + } +} + +func TestFromLocalIsInputOrderIndependent(t *testing.T) { + specs := testSpecs() + a, err := FromLocal(releaseInput(specs)) + if err != nil { + t.Fatal(err) + } + rev := make([]specFile, len(specs)) + for i := range specs { + rev[len(specs)-1-i] = specs[i] + } + b, err := FromLocal(releaseInput(rev)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(mustEncode(t, a), mustEncode(t, b)) { + t.Fatal("file order changed the bundle") + } +} + +func TestFromLocalNotShareable(t *testing.T) { + tests := []struct { + name string + mutate func(in *ReleaseInput) + reason ShareReason + }{ + {"rclone encrypted file", func(in *ReleaseInput) { + m := in.Files[0].Meta + m.Encryption, m.Password, m.Salt = metapb.Encryption_RCLONE, "RCLONEPW", "RCLONESALT" + }, ReasonRclone}, + {"header encryption", func(in *ReleaseInput) { in.Files[0].Meta.Encryption = metapb.Encryption_HEADERS }, ReasonOtherEncryption}, + {"aes file whose password is not from the nzb", func(in *ReleaseInput) { + for i := range in.Files { + if len(in.Files[i].Meta.AesKey) > 0 { + in.Files[i].PasswordFromNZB = false + } + } + }, ReasonPasswordNotNZB}, + {"nested encrypted outer whose password is not from the nzb", func(in *ReleaseInput) { + for i := range in.Files { + if len(in.Files[i].Meta.SharedOuterSources) > 0 && len(in.Files[i].Meta.SharedOuterSources[0].AesKey) > 0 { + in.Files[i].PasswordFromNZB = false + } + } + }, ReasonPasswordNotNZB}, + {"healed by local par2 patches", func(in *ReleaseInput) { in.Files[0].PatchedLocally = true }, ReasonLocalPatch}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + in := releaseInput(testSpecs()) + tc.mutate(&in) + _, err := FromLocal(in) + if !errors.Is(err, ErrNotShareable) { + t.Fatalf("err = %v, want ErrNotShareable", err) + } + var ns *NotShareableError + if !errors.As(err, &ns) || ns.Reason != tc.reason { + t.Fatalf("reason = %+v, want %s", ns, tc.reason) + } + }) + } +} + +func TestFromLocalPlainFileNeedsNoPasswordFlag(t *testing.T) { + in := releaseInput([]specFile{testSpecs()[0]}) + in.Files[0].PasswordFromNZB = false + if _, err := FromLocal(in); err != nil { + t.Fatalf("plain file with the flag unset: %v", err) + } +} + +func TestEncodedBundleNeverHoldsLocalSecrets(t *testing.T) { + in := releaseInput(testSpecs()) + secrets := []string{"RCLONEPW", "RCLONESALT", "/local/store/path.nzbz", "store-ref-abc", "dl-id-123", + "provider-fingerprint-secret", "usenet.provider.example", "radarr-main", "NZBGeek", "movies-cat"} + for i := range in.Files { + m := in.Files[i].Meta + // A stray password and salt on a non-rclone file must not travel. + m.Password, m.Salt = "RCLONEPW", "RCLONESALT" + m.StoreRef = "store-ref-abc" + m.SourceNzbPath = "/local/store/path.nzbz" + } + in.NZBPassword = "nzb-pass" + b, err := FromLocal(in) + if err != nil { + t.Fatal(err) + } + raw := mustEncode(t, b) + wire, err := Compress(raw) + if err != nil { + t.Fatal(err) + } + for _, s := range secrets { + if bytes.Contains(raw, []byte(s)) { + t.Errorf("canonical bytes contain %q", s) + } + } + for _, ts := range []int64{1700000001} { // local modified/created time + if bytes.Contains(raw, appendVarintBytes(ts)) { + t.Errorf("canonical bytes contain the local timestamp %d", ts) + } + } + if !bytes.Contains(raw, []byte("nzb-pass")) { + t.Error("the NZB password was dropped") + } + _ = wire +} + +func appendVarintBytes(v int64) []byte { + var b []byte + u := uint64(v) + for u >= 0x80 { + b = append(b, byte(u)|0x80) + u >>= 7 + } + return append(b, byte(u)) +} + +func TestFromLocalNames(t *testing.T) { + base := testSpecs()[0] + mk := func(name string, id string) specFile { + f := base + f.name = name + f.par2 = nil + f.segs = []specSeg{full(id+"1@x.test", 1000), full(id+"2@x.test", 1000), full(id+"3@x.test", 500)} + return f + } + in := releaseInput([]specFile{mk("a.mkv", "a"), mk("a.mkv", "b"), mk("A.MKV", "c"), mk("../../etc/x.mkv", "d"), mk("CON", "e")}) + b, err := FromLocal(in) + if err != nil { + t.Fatal(err) + } + var names []string + for _, f := range b.Files { + names = append(names, f.Name) + } + seen := map[string]bool{} + for _, n := range names { + if seen[strings.ToLower(n)] { + t.Errorf("duplicate name %q in %v", n, names) + } + seen[strings.ToLower(n)] = true + if err := checkFilename(n); err != nil { + t.Errorf("name %q: %v", n, err) + } + } + for _, want := range []string{"x.mkv", "_CON"} { + if !seen[strings.ToLower(want)] { + t.Errorf("missing %q in %v", want, names) + } + } + // Deterministic whatever the input order. + in2 := releaseInput([]specFile{mk("CON", "e"), mk("../../etc/x.mkv", "d"), mk("A.MKV", "c"), mk("a.mkv", "b"), mk("a.mkv", "a")}) + b2, err := FromLocal(in2) + if err != nil || !bytes.Equal(mustEncode(t, b), mustEncode(t, b2)) { + t.Fatalf("name disambiguation depends on input order (err %v)", err) + } +} + +func TestFromLocalNameFallsBackToOriginalFilename(t *testing.T) { + in := releaseInput([]specFile{testSpecs()[0]}) + in.Files[0].Name = "" + b, err := FromLocal(in) + if err != nil || b.Files[0].Name != "movie.mkv" { + t.Fatalf("%v %v", err, b) + } +} + +func TestFromLocalRejects(t *testing.T) { + tests := []struct { + name string + mutate func(in *ReleaseInput) + want error + }{ + {"nil metadata", func(in *ReleaseInput) { in.Files[0].Meta = nil }, ErrInvalidField}, + {"no files", func(in *ReleaseInput) { in.Files = nil }, ErrInvalidField}, + {"empty title", func(in *ReleaseInput) { in.Title = " \u202E " }, ErrInvalidField}, + {"no post date", func(in *ReleaseInput) { in.PostedAt = time.Time{} }, ErrInvalidField}, + {"unknown scheme", func(in *ReleaseInput) { in.Scheme = 0 }, ErrUnknownScheme}, + {"unresolved segment refs", func(in *ReleaseInput) { + m := in.Files[0].Meta + m.SegmentData = nil + m.SegmentRefs = []*metapb.SegmentRef{{StoreIndex: 0, EndOffset: 9}} + }, ErrUnresolved}, + {"unresolved segment runs", func(in *ReleaseInput) { + m := in.Files[0].Meta + m.SegmentData = nil + m.SegmentRuns = []*metapb.SegmentRun{{Count: 3}} + }, ErrUnresolved}, + {"inconsistent local metadata", func(in *ReleaseInput) { in.Files[0].Meta.FileSize++ }, ErrSizeMismatch}, + {"bad message id", func(in *ReleaseInput) { in.Files[0].Meta.SegmentData[0].Id = "a b@x" }, ErrBadMessageID}, + {"empty message id", func(in *ReleaseInput) { in.Files[0].Meta.SegmentData[0].Id = "" }, ErrBadMessageID}, + {"shared outer index out of range", func(in *ReleaseInput) { + for _, f := range in.Files { + for _, ns := range f.Meta.NestedSources { + ns.SharedOuterSourceIndex = 9 + } + } + }, ErrOutOfRange}, + {"same file twice", func(in *ReleaseInput) { + dup := in.Files[0] + dup.Meta = specFileMetaClone(in.Files[0].Meta) + dup.Meta.OriginalFilename = "other.mkv" + dup.Name = "other.mkv" + in.Files = append(in.Files, dup) + }, ErrDuplicateFile}, + {"segment missing from the nzb list", func(in *ReleaseInput) { + in.Scheme, in.NZBFiles = SchemeNZB, nzbLayout()[1:] + }, ErrSegmentNotInNZB}, + {"nzb scheme without a list", func(in *ReleaseInput) { in.Scheme, in.NZBFiles = SchemeNZB, nil }, ErrBadPool}, + {"bad id in the nzb list", func(in *ReleaseInput) { + in.Scheme, in.NZBFiles = SchemeNZB, nzbLayout() + in.NZBFiles[5][0] = "bad id" + }, ErrBadMessageID}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + in := releaseInput(testSpecs()) + tc.mutate(&in) + if _, err := FromLocal(in); !errors.Is(err, tc.want) { + t.Fatalf("err = %v, want %v", err, tc.want) + } + }) + } +} + +func specFileMetaClone(m *metapb.FileMetadata) *metapb.FileMetadata { + return proto.Clone(m).(*metapb.FileMetadata) +} + +func TestFromLocalAcceptsBracketedIDs(t *testing.T) { + in := releaseInput(testSpecs()) + for _, f := range in.Files { + for _, s := range f.Meta.SegmentData { + s.Id = "<" + s.Id + ">" + } + } + b, err := FromLocal(in) + if err != nil { + t.Fatal(err) + } + want := goodBundle(t) + if !bytes.Equal(mustEncode(t, b), mustEncode(t, want)) { + t.Fatal("bracketed ids changed the bundle") + } +} + +func TestFromLocalDoesNotMutateInput(t *testing.T) { + in := releaseInput(testSpecs()) + before := make([][]byte, len(in.Files)) + for i, f := range in.Files { + before[i] = []byte(f.Meta.String()) + } + if _, err := FromLocal(in); err != nil { + t.Fatal(err) + } + for i, f := range in.Files { + if string(before[i]) != f.Meta.String() { + t.Fatalf("file %d metadata mutated", i) + } + } +} diff --git a/internal/peernet/bundle/compress.go b/internal/peernet/bundle/compress.go new file mode 100644 index 000000000..4be046569 --- /dev/null +++ b/internal/peernet/bundle/compress.go @@ -0,0 +1,108 @@ +package bundle + +import ( + "bytes" + "errors" + "fmt" + "io" + + "github.com/klauspost/compress/zstd" +) + +// Compression: zstd at its highest level, single-threaded, with a window of +// MaxWindowSize. No dictionary is used: the bulk of a bundle is high-entropy +// message-IDs, which a dictionary cannot predict, while the repetitive layout +// (sizes, offsets, references) is run-length coded in the schema itself and +// costs a few bytes per file. See TestBundleSizes for the measured sizes. + +// Compress compresses canonical bundle bytes for the wire. The result never +// exceeds MaxCompressedSize. +func Compress(canonical []byte) ([]byte, error) { + if len(canonical) > MaxDecompressedSize { + return nil, fmt.Errorf("%w: %d bytes uncompressed", ErrTooLarge, len(canonical)) + } + enc, err := zstd.NewWriter(nil, + zstd.WithEncoderLevel(zstd.SpeedBestCompression), + zstd.WithEncoderConcurrency(1), + zstd.WithWindowSize(MaxWindowSize), + zstd.WithEncoderCRC(true), + ) + if err != nil { + return nil, fmt.Errorf("bundle: zstd encoder: %w", err) + } + defer func() { _ = enc.Close() }() + out := enc.EncodeAll(canonical, nil) + if len(out) > MaxCompressedSize { + return nil, fmt.Errorf("%w: %d bytes compressed", ErrTooLarge, len(out)) + } + return out, nil +} + +// Decompress returns the canonical bytes of a wire bundle. The input may not +// exceed MaxCompressedSize, a frame may not ask for a window above +// MaxWindowSize, and decoding stops with ErrTooLarge as soon as the output +// would pass MaxDecompressedSize, so a tiny input cannot make it allocate a +// large buffer. Decoding is synchronous: no goroutine outlives the call. +func Decompress(wire []byte) ([]byte, error) { + if len(wire) > MaxCompressedSize { + return nil, fmt.Errorf("%w: %d bytes compressed", ErrTooLarge, len(wire)) + } + if len(wire) == 0 { + return nil, fmt.Errorf("%w: empty input", ErrMalformed) + } + dec, err := zstd.NewReader(bytes.NewReader(wire), + zstd.WithDecoderConcurrency(1), + zstd.WithDecoderMaxWindow(MaxWindowSize), + zstd.WithDecoderMaxMemory(MaxDecompressedSize), + zstd.WithDecoderLowmem(true), + ) + if err != nil { + return nil, fmt.Errorf("%w: zstd: %v", ErrMalformed, err) + } + defer dec.Close() + out, err := readBounded(dec) + switch { + case len(out) > MaxDecompressedSize: + return nil, fmt.Errorf("%w: more than %d bytes uncompressed", ErrTooLarge, MaxDecompressedSize) + case errors.Is(err, zstd.ErrDecoderSizeExceeded), errors.Is(err, zstd.ErrWindowSizeExceeded): + return nil, fmt.Errorf("%w: zstd limits: %v", ErrTooLarge, err) + case err != nil: + return nil, fmt.Errorf("%w: zstd: %v", ErrMalformed, err) + case len(out) == 0: + return nil, fmt.Errorf("%w: no data", ErrMalformed) + } + return out, nil +} + +// readBounded reads r in fixed chunks, up to MaxDecompressedSize+1 bytes, and +// joins them once, so peak memory stays near twice the output rather than the +// several times that a growing buffer costs. +func readBounded(r io.Reader) ([]byte, error) { + const chunk = 1 << 20 + var chunks [][]byte + total := 0 + cur := make([]byte, 0, chunk) + var err error + for total <= MaxDecompressedSize { + if len(cur) == cap(cur) { + chunks = append(chunks, cur) + cur = make([]byte, 0, min(chunk, MaxDecompressedSize+1-total)) + } + var n int + n, err = r.Read(cur[len(cur):cap(cur)]) + cur = cur[:len(cur)+n] + total += n + if err != nil { + break + } + } + if errors.Is(err, io.EOF) { + err = nil + } + chunks = append(chunks, cur) + out := make([]byte, 0, total) + for _, c := range chunks { + out = append(out, c...) + } + return out, err +} diff --git a/internal/peernet/bundle/compress_test.go b/internal/peernet/bundle/compress_test.go new file mode 100644 index 000000000..9a4352e4b --- /dev/null +++ b/internal/peernet/bundle/compress_test.go @@ -0,0 +1,166 @@ +package bundle + +import ( + "bytes" + "crypto/rand" + "errors" + "io" + "runtime" + "testing" + "time" + + "github.com/klauspost/compress/zstd" +) + +// zeroBomb returns a valid zstd stream that expands to n zero bytes. +func zeroBomb(t testing.TB, n int, window int) []byte { + t.Helper() + var buf bytes.Buffer + opts := []zstd.EOption{zstd.WithEncoderConcurrency(1), zstd.WithEncoderLevel(zstd.SpeedFastest)} + if window > 0 { + opts = append(opts, zstd.WithWindowSize(window)) + } + enc, err := zstd.NewWriter(&buf, opts...) + if err != nil { + t.Fatal(err) + } + chunk := make([]byte, 1<<20) + for left := n; left > 0; { + c := min(left, len(chunk)) + if _, err := enc.Write(chunk[:c]); err != nil { + t.Fatal(err) + } + left -= c + } + if err := enc.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestCompressRoundTrip(t *testing.T) { + raw := mustEncode(t, goodBundle(t)) + wire, err := Compress(raw) + if err != nil { + t.Fatalf("Compress: %v", err) + } + got, err := Decompress(wire) + if err != nil || !bytes.Equal(got, raw) { + t.Fatalf("Decompress: %v equal=%v", err, bytes.Equal(got, raw)) + } +} + +func TestDecompressRejects(t *testing.T) { + good, _ := Compress([]byte("hello hello hello hello")) + tests := []struct { + name string + in []byte + want error + }{ + {"empty", nil, ErrMalformed}, + {"garbage", []byte("this is not zstd at all, just text"), ErrMalformed}, + {"truncated frame", good[:len(good)-2], ErrMalformed}, + {"input over 8 MB", make([]byte, MaxCompressedSize+1), ErrTooLarge}, + {"bomb over 64 MB", zeroBomb(t, MaxDecompressedSize+1, 0), ErrTooLarge}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if _, err := Decompress(tc.in); !errors.Is(err, tc.want) { + t.Fatalf("Decompress = %v, want %v", err, tc.want) + } + }) + } +} + +func TestDecompressAcceptsExactLimit(t *testing.T) { + out, err := Decompress(zeroBomb(t, MaxDecompressedSize, 0)) + if err != nil || len(out) != MaxDecompressedSize { + t.Fatalf("len %d err %v", len(out), err) + } +} + +func TestDecompressBombMemoryIsBounded(t *testing.T) { + bomb := zeroBomb(t, 512<<20, 0) // 512 MiB of zeros in a few KB + if len(bomb) > 1<<20 { + t.Fatalf("bomb is %d bytes, expected tiny", len(bomb)) + } + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + _, err := Decompress(bomb) + runtime.ReadMemStats(&after) + if !errors.Is(err, ErrTooLarge) { + t.Fatalf("got %v", err) + } + if delta := after.TotalAlloc - before.TotalAlloc; delta > 4*MaxDecompressedSize { + t.Fatalf("allocated %d bytes for a bomb; limit %d", delta, 4*MaxDecompressedSize) + } +} + +func TestDecompressRejectsOversizedWindow(t *testing.T) { + // A frame that asks the decoder for a 128 MB window must be refused even + // though its content is small. + data := zeroBomb(t, 1<<20, 128<<20) + if _, err := Decompress(data); err == nil { + t.Fatal("128 MB window accepted") + } +} + +func TestCompressRejectsTooLarge(t *testing.T) { + if _, err := Compress(make([]byte, MaxDecompressedSize+1)); !errors.Is(err, ErrTooLarge) { + t.Fatalf("oversized input: %v", err) + } + noise := make([]byte, MaxCompressedSize+(1<<20)) + if _, err := io.ReadFull(rand.Reader, noise); err != nil { + t.Fatal(err) + } + if _, err := Compress(noise); !errors.Is(err, ErrTooLarge) { + t.Fatalf("incompressible output over 8 MB: %v", err) + } +} + +func TestDecompressLeavesNoGoroutines(t *testing.T) { + wire, err := Compress(mustEncode(t, goodBundle(t))) + if err != nil { + t.Fatal(err) + } + runtime.GC() + before := runtime.NumGoroutine() + for i := 0; i < 200; i++ { + if _, err := Decompress(wire); err != nil { + t.Fatal(err) + } + _, _ = Decompress([]byte("junk")) + _, _ = Decompress(zeroBomb(t, MaxDecompressedSize+1, 0)[:100]) + } + deadline := time.Now().Add(2 * time.Second) + for runtime.NumGoroutine() > before && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if after := runtime.NumGoroutine(); after > before { + t.Fatalf("goroutines %d -> %d", before, after) + } +} + +// TestBundleSizes documents the measured size of typical releases. Message-IDs +// are random, so they dominate; layout (sizes, offsets, references) is run +// length coded and costs a few bytes per file. +func TestBundleSizes(t *testing.T) { + for _, tc := range []struct { + name string + gb int64 + maxWire int + }{{"1GB", 1, 90 << 10}, {"50GB", 50, 3 << 20}} { + n := int(tc.gb<<30/716_800) + 1 + b := syntheticBundle(t, n, 716_800) + raw := mustEncode(t, b) + wire, err := Compress(raw) + if err != nil { + t.Fatal(err) + } + t.Logf("%s: %d segments, canonical %d bytes, compressed %d bytes", tc.name, n, len(raw), len(wire)) + if len(wire) > tc.maxWire { + t.Fatalf("%s: compressed %d bytes, want <= %d", tc.name, len(wire), tc.maxWire) + } + } +} diff --git a/internal/peernet/bundle/convert.go b/internal/peernet/bundle/convert.go new file mode 100644 index 000000000..21c30ca0b --- /dev/null +++ b/internal/peernet/bundle/convert.go @@ -0,0 +1,136 @@ +package bundle + +import ( + "encoding/hex" + "fmt" + "time" + + "github.com/kipsilabs/altmount/internal/holes" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +// LocalFile is one file ready for the metadata service. Name is a sanitised +// base name: the caller derives the virtual path locally and never from +// anything the peer sent. +type LocalFile struct { + Name string + Meta *metapb.FileMetadata +} + +// Local is a bundle converted to the local on-disk model. +type Local struct { + Scheme Scheme + ReleaseID contentid.ID + Title string + PostedUnix int64 + NZBPassword string + // NZBFiles (SchemeNZB) is the full release segment list per NZB file, for + // computing the Release ID of the local import later. It lacks subjects, + // posters, groups and encoded sizes, so it cannot regenerate an NZB. + NZBFiles [][]string + Files []LocalFile +} + +// Field-by-field mapping, bundle to local metadata. +// +// shared: FileSize, SegmentData (id, size, start, end), ReleaseDate, +// OriginalFilename, Encryption=AES + AesKey/AesIv, NestedSources and +// SharedOuterSources (segments, key, iv, inner offset/length/volume +// size), ClipBoundaries, Par2Files (name, size, segments) +// derived: Status (HEALTHY, as a fresh import), CreatedAt/ModifiedAt (now), +// placeholder ids of gaps +// dropped: SourceNzbPath, StoreRef, NzbdavId, Password, Salt, KnownHoles, +// HoleProviderFingerprint, SegmentRefs/Runs (inline segments are +// used instead), segment-store references +// +// The result is the v1 inline layout: the readers need no .nzbz store entry. + +func gapSalt(id contentid.ID, tag string) string { return hex.EncodeToString(id[:8]) + tag } + +func segmentData(pool []string, segs []Segment, salt string) []*metapb.SegmentData { + if len(segs) == 0 { + return nil + } + out := make([]*metapb.SegmentData, len(segs)) + for i, s := range segs { + id := "" + if s.Ref == GapRef { + id = holes.PlaceholderID(i+1, salt) + } else { + id = pool[s.Ref] + } + out[i] = &metapb.SegmentData{Id: id, SegmentSize: s.Size, StartOffset: s.Start, EndOffset: s.End} + } + return out +} + +// buildMeta renders one (validated) file as on-disk metadata without local +// timestamps. Verify uses it to recompute the File ID with contentid. +func buildMeta(b *Bundle, f *File) *metapb.FileMetadata { + m := &metapb.FileMetadata{ + FileSize: f.Size, + Status: metapb.FileStatus_FILE_STATUS_HEALTHY, + ReleaseDate: f.ReleaseDate, + OriginalFilename: f.Name, + SegmentData: segmentData(b.Pool, f.Segments, gapSalt(f.FileID, "")), + } + if len(f.AESKey) > 0 { + m.Encryption = metapb.Encryption_AES + m.AesKey = append([]byte(nil), f.AESKey...) + m.AesIv = append([]byte(nil), f.AESIV...) + } + for i, o := range f.Outer { + sh := &metapb.NestedSegmentSource{ + Segments: segmentData(b.Pool, o.Segments, gapSalt(f.FileID, fmt.Sprintf("o%d", i))), + InnerVolumeSize: o.VolumeSize, + } + if len(o.AESKey) > 0 { + sh.AesKey = append([]byte(nil), o.AESKey...) + sh.AesIv = append([]byte(nil), o.AESIV...) + } + m.SharedOuterSources = append(m.SharedOuterSources, sh) + } + for _, e := range f.Extents { + m.NestedSources = append(m.NestedSources, &metapb.NestedSegmentSource{ + SharedOuterSourceIndex: int32(e.Outer + 1), + InnerOffset: e.InnerOffset, + InnerLength: e.InnerLength, + }) + } + for _, c := range f.Clips { + m.ClipBoundaries = append(m.ClipBoundaries, &metapb.ClipBoundary{ByteLen: c.ByteLen, Delta_90K: c.Delta90k}) + } + for i, p := range f.Par2 { + m.Par2Files = append(m.Par2Files, &metapb.Par2FileReference{ + Filename: p.Name, + FileSize: p.Size, + SegmentData: segmentData(b.Pool, p.Segments, gapSalt(f.FileID, fmt.Sprintf("p%d", i))), + }) + } + return m +} + +// ToLocal validates b and converts it to local metadata messages, ready for +// MetadataService.WriteFileMetadata. now stamps CreatedAt and ModifiedAt. +// Virtual paths are never taken from the bundle. +func ToLocal(b *Bundle, now time.Time) (*Local, error) { + if err := Validate(b); err != nil { + return nil, err + } + loc := &Local{Scheme: b.Scheme, ReleaseID: b.ReleaseID, Title: b.Title, PostedUnix: b.PostedUnix, NZBPassword: b.NZBPassword} + if b.Scheme == SchemeNZB { + pos := 0 + for _, n := range b.ReleaseFiles { + loc.NZBFiles = append(loc.NZBFiles, append([]string{}, b.Pool[pos:pos+n]...)) + pos += n + } + } + for i := range b.Files { + f := &b.Files[i] + m := buildMeta(b, f) + m.CreatedAt, m.ModifiedAt = now.Unix(), now.Unix() + loc.Files = append(loc.Files, LocalFile{Name: f.Name, Meta: m}) + } + return loc, nil +} diff --git a/internal/peernet/bundle/convert_test.go b/internal/peernet/bundle/convert_test.go new file mode 100644 index 000000000..bf153eaa3 --- /dev/null +++ b/internal/peernet/bundle/convert_test.go @@ -0,0 +1,203 @@ +package bundle + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/kipsilabs/altmount/internal/holes" + "github.com/kipsilabs/altmount/internal/metadata" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +type effSeg struct { + ID string + Size, Start, End int64 +} + +type effNested struct { + Segs []effSeg + Key, IV string + Offset, Length int64 + VolumeSize int64 +} + +type effPar2 struct { + Name string + Size int64 + Segs []effSeg +} + +// effective is what the readers act on, with local-only fields left out and +// gap placeholders normalised (their ids are local by design). +type effective struct { + Name string + Size int64 + Date int64 + Encryption metapb.Encryption + Key, IV string + Segs []effSeg + Nested []effNested + Clips []Clip + Par2 []effPar2 +} + +func effSegs(in []*metapb.SegmentData) []effSeg { + var out []effSeg + for _, s := range in { + id := s.Id + if holes.IsPlaceholderID(id) { + id = "GAP" + } + out = append(out, effSeg{id, s.SegmentSize, s.StartOffset, s.EndOffset}) + } + return out +} + +func effectiveOf(t *testing.T, m *metapb.FileMetadata) effective { + t.Helper() + require.NoError(t, expandShared(m)) + e := effective{Name: m.OriginalFilename, Size: m.FileSize, Date: m.ReleaseDate, Encryption: m.Encryption, + Key: string(m.AesKey), IV: string(m.AesIv), Segs: effSegs(m.SegmentData)} + for _, ns := range m.NestedSources { + e.Nested = append(e.Nested, effNested{effSegs(ns.Segments), string(ns.AesKey), string(ns.AesIv), ns.InnerOffset, ns.InnerLength, ns.InnerVolumeSize}) + } + for _, c := range m.ClipBoundaries { + e.Clips = append(e.Clips, Clip{c.ByteLen, c.Delta_90K}) + } + for _, p := range m.Par2Files { + e.Par2 = append(e.Par2, effPar2{p.Filename, p.FileSize, effSegs(p.SegmentData)}) + } + return e +} + +func TestToLocalRoundTripFieldByField(t *testing.T) { + b := goodBundle(t) + loc, err := ToLocal(b, testNow) + require.NoError(t, err) + require.Len(t, loc.Files, len(testSpecs())) + got := map[string]*metapb.FileMetadata{} + for _, f := range loc.Files { + got[f.Name] = f.Meta + } + for _, s := range testSpecs() { + m, ok := got[s.name] + require.True(t, ok, s.name) + assert.Equal(t, effectiveOf(t, s.meta()), effectiveOf(t, m), s.name) + } +} + +func TestToLocalDropsLocalFieldsAndIsNeverAPath(t *testing.T) { + loc, err := ToLocal(goodBundle(t), testNow) + require.NoError(t, err) + for _, f := range loc.Files { + m := f.Meta + assert.Empty(t, m.SourceNzbPath) + assert.Empty(t, m.StoreRef) + assert.Empty(t, m.NzbdavId) + assert.Empty(t, m.Password) + assert.Empty(t, m.Salt) + assert.Empty(t, m.KnownHoles) + assert.Empty(t, m.HoleProviderFingerprint) + assert.Empty(t, m.SegmentRefs) + assert.Empty(t, m.SegmentRuns) + assert.Equal(t, metapb.FileStatus_FILE_STATUS_HEALTHY, m.Status) + assert.Equal(t, testNow.Unix(), m.CreatedAt) + assert.Equal(t, testNow.Unix(), m.ModifiedAt) + assert.False(t, strings.ContainsAny(f.Name, `/\`), f.Name) + assert.Equal(t, f.Name, m.OriginalFilename) + } +} + +func TestToLocalPreservesFileIdentity(t *testing.T) { + b := goodBundle(t) + loc, err := ToLocal(b, testNow) + require.NoError(t, err) + want := map[string]contentid.ID{} + for _, f := range b.Files { + want[f.Name] = f.FileID + } + for _, f := range loc.Files { + require.NoError(t, expandShared(f.Meta)) + id, err := contentid.FileID(f.Meta) + require.NoError(t, err) + assert.Equal(t, want[f.Name], id, f.Name) + } +} + +func TestToLocalSharesOuterSources(t *testing.T) { + loc, err := ToLocal(goodBundle(t), testNow) + require.NoError(t, err) + for _, f := range loc.Files { + if f.Name != "nested.mkv" { + continue + } + assert.Len(t, f.Meta.SharedOuterSources, 1) + assert.Len(t, f.Meta.NestedSources, 2) + for _, ns := range f.Meta.NestedSources { + assert.Equal(t, int32(1), ns.SharedOuterSourceIndex) + } + } +} + +func TestToLocalDoesNotAliasTheBundle(t *testing.T) { + b := goodBundle(t) + want := mustEncode(t, b) + loc, err := ToLocal(b, testNow) + require.NoError(t, err) + for _, f := range loc.Files { + for i := range f.Meta.AesKey { + f.Meta.AesKey[i] = 0 + } + for _, o := range f.Meta.SharedOuterSources { + for i := range o.AesKey { + o.AesKey[i] = 0 + } + } + } + assert.Equal(t, want, encodeUnchecked(b)) +} + +func TestToLocalNZBSchemeCarriesTheReleaseList(t *testing.T) { + loc, err := ToLocal(goodNZBBundle(t), testNow) + require.NoError(t, err) + assert.Equal(t, SchemeNZB, loc.Scheme) + assert.Equal(t, nzbLayout(), loc.NZBFiles) + loc, err = ToLocal(goodBundle(t), testNow) + require.NoError(t, err) + assert.Nil(t, loc.NZBFiles) +} + +func TestToLocalRejectsInvalidBundle(t *testing.T) { + b := goodBundle(t) + b.Files[0].Size++ + _, err := ToLocal(b, testNow) + assert.ErrorIs(t, err, ErrSizeMismatch) + _, err = ToLocal(nil, testNow) + assert.ErrorIs(t, err, ErrInvalidField) +} + +// The converted metadata is read by the real metadata service from inline +// segments alone: no .nzbz store entry is needed. +func TestToLocalIsReadableWithoutAStore(t *testing.T) { + loc, err := ToLocal(goodBundle(t), testNow) + require.NoError(t, err) + svc := metadata.NewMetadataService(t.TempDir()) + want := map[string]contentid.ID{} + for _, f := range goodBundle(t).Files { + want[f.Name] = f.FileID + } + for _, f := range loc.Files { + require.NoError(t, svc.WriteFileMetadata("/release/"+f.Name, f.Meta)) + got, err := svc.ReadFileMetadata("/release/" + f.Name) + require.NoError(t, err) + require.NotNil(t, got) + id, err := contentid.FileID(got) + require.NoError(t, err) + assert.Equal(t, want[f.Name], id, f.Name) + assert.Equal(t, f.Meta.FileSize, got.FileSize) + } +} diff --git a/internal/peernet/bundle/coverage_test.go b/internal/peernet/bundle/coverage_test.go new file mode 100644 index 000000000..95d643772 --- /dev/null +++ b/internal/peernet/bundle/coverage_test.go @@ -0,0 +1,71 @@ +package bundle + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// refs returns the pool indices of the given ids. +func poolIdx(t *testing.T, b *Bundle, id string) int64 { + t.Helper() + for i, p := range b.Pool { + if p == id { + return int64(i) + } + } + t.Fatalf("%s not in pool", id) + return 0 +} + +func TestNZBCoverageRules(t *testing.T) { + t.Run("good bundle passes", func(t *testing.T) { + require.NoError(t, Validate(goodNZBBundle(t))) + }) + bad := map[string]func(t *testing.T, b *Bundle){ + "segment used twice in one file": func(t *testing.T, b *Bundle) { + f := fileNamed(t, b, "movie.mkv") + f.Segments[2].Ref = f.Segments[1].Ref + }, + "segments out of NZB order": func(t *testing.T, b *Bundle) { + f := fileNamed(t, b, "movie.mkv") + f.Segments[0].Ref, f.Segments[1].Ref = f.Segments[1].Ref, f.Segments[0].Ref + }, + "file covers only part of its NZB entry": func(t *testing.T, b *Bundle) { + f := fileNamed(t, b, "movie.mkv") + f.Segments = f.Segments[:2] + f.Size = 2000 + b.TotalSize -= 500 + }, + "file spans two NZB entries": func(t *testing.T, b *Bundle) { + f := fileNamed(t, b, "movie.mkv") + f.Segments[2].Ref = poolIdx(t, b, "p1@x.test") + f.Segments[2].Size, f.Segments[2].End = 500, 499 + }, + "one entry read by two inline files": func(t *testing.T, b *Bundle) { + a, c := fileNamed(t, b, "movie.mkv"), fileNamed(t, b, "secret.bin") + c.Segments = append([]Segment(nil), a.Segments...) + c.AESKey, c.AESIV = nil, nil + c.Size = a.Size + b.TotalSize += a.Size - 100 + }, + "outer source is not a whole entry": func(t *testing.T, b *Bundle) { + o := &fileNamed(t, b, "disc.m2ts").Outer[0] + o.Segments = o.Segments[:2] + }, + } + for name, mutate := range bad { + t.Run(name, func(t *testing.T) { + b := clone(goodNZBBundle(t)) + mutate(t, b) + err := Validate(b) + require.Error(t, err) + require.ErrorIs(t, err, ErrCoverage) + }) + } + t.Run("one par2 entry listed by two files is fine", func(t *testing.T) { + b := clone(goodNZBBundle(t)) + fileNamed(t, b, "secret.bin").Par2 = append([]Par2File(nil), fileNamed(t, b, "movie.mkv").Par2...) + require.NoError(t, Validate(b)) + }) +} diff --git a/internal/peernet/bundle/fuzz_test.go b/internal/peernet/bundle/fuzz_test.go new file mode 100644 index 000000000..5f95651bc --- /dev/null +++ b/internal/peernet/bundle/fuzz_test.go @@ -0,0 +1,167 @@ +package bundle + +import ( + "bytes" + "testing" +) + +func seedBundles(t testing.TB) [][]byte { + t.Helper() + var out [][]byte + for _, b := range []*Bundle{goodBundle(t), goodNZBBundle(t), syntheticBundle(t, 50, 716_800)} { + out = append(out, mustEncode(t, b)) + } + return out +} + +func FuzzDecodeBundle(f *testing.F) { + for _, s := range seedBundles(f) { + f.Add(s) + f.Add(s[:len(s)/2]) + } + f.Add([]byte{}) + f.Add([]byte{0x08, 0x01}) + f.Add([]byte{0x08, 0x81, 0x00}) + f.Fuzz(func(t *testing.T, data []byte) { + b, err := Decode(data) + if err != nil { + return + } + again, err := Encode(b) + if err != nil { + t.Fatalf("decoded bundle does not re-encode: %v", err) + } + if !bytes.Equal(again, data) { + t.Fatal("decode then encode changed the bytes") + } + if _, err := ToLocal(b, testNow); err != nil { + t.Fatalf("valid bundle does not convert: %v", err) + } + }) +} + +func FuzzDecompress(f *testing.F) { + for _, s := range seedBundles(f) { + w, err := Compress(s) + if err != nil { + f.Fatal(err) + } + f.Add(w) + f.Add(w[:len(w)/2]) + } + f.Add(zeroBomb(f, MaxDecompressedSize+1, 0)) + f.Add(zeroBomb(f, 1<<20, 128<<20)) + f.Add([]byte("not zstd")) + f.Add([]byte{}) + f.Add([]byte{0x28, 0xb5, 0x2f, 0xfd}) + f.Fuzz(func(t *testing.T, data []byte) { + out, err := Decompress(data) + if err != nil { + if out != nil { + t.Fatal("output returned with an error") + } + return + } + if len(out) == 0 || len(out) > MaxDecompressedSize || len(data) > MaxCompressedSize { + t.Fatalf("limits violated: in %d out %d", len(data), len(out)) + } + }) +} + +// mutateBundle breaks one field of a good bundle, chosen by sel. +func mutateBundle(b *Bundle, sel uint8, v int64) { + f := &b.Files[int(sel/32)%len(b.Files)] + switch sel % 16 { + case 0: + f.Size = v + case 1: + if len(f.Segments) > 0 { + f.Segments[0].Start = v + } + case 2: + if len(f.Segments) > 0 { + f.Segments[len(f.Segments)-1].End = v + } + case 3: + if len(f.Segments) > 0 { + f.Segments[0].Size = v + } + case 4: + if len(f.Segments) > 0 { + f.Segments[0].Ref = v + } + case 5: + if len(f.Extents) > 0 { + f.Extents[0].InnerOffset = v + } + case 6: + if len(f.Extents) > 0 { + f.Extents[len(f.Extents)-1].InnerLength = v + } + case 7: + if len(f.Extents) > 0 { + f.Extents[0].Outer = int(v) + } + case 8: + if len(f.Outer) > 0 { + f.Outer[0].VolumeSize = v + } + case 9: + if len(f.Clips) > 0 { + f.Clips[0].ByteLen = v + } + case 10: + if len(f.Clips) > 0 { + f.Clips[0].Delta90k = v + } + case 11: + b.TotalSize = v + case 12: + f.ReleaseDate = v + case 13: + b.PostedUnix = v + case 14: + if len(f.Outer) > 0 && len(f.Outer[0].Segments) > 0 { + f.Outer[0].Segments[0].End = v + } + case 15: + if len(f.Par2) > 0 && len(f.Par2[0].Segments) > 0 { + f.Par2[0].Segments[0].Size = v + } + } +} + +func FuzzValidate(f *testing.F) { + for _, s := range seedBundles(f) { + f.Add(s, uint8(0), int64(0)) + } + f.Add([]byte{}, uint8(3), int64(-1)) + f.Add([]byte{}, uint8(36), int64(1<<62)) + f.Add([]byte{}, uint8(70), int64(576)) + f.Fuzz(func(t *testing.T, data []byte, sel uint8, v int64) { + check := func(b *Bundle) { + if err := Validate(b); err != nil { + return + } + raw, err := Encode(b) + if err != nil { + t.Fatalf("valid bundle does not encode: %v", err) + } + got, err := Decode(raw) + if err != nil { + t.Fatalf("valid bundle does not decode: %v", err) + } + if again, err := Encode(got); err != nil || !bytes.Equal(again, raw) { + t.Fatalf("canonical re-encode differs (%v)", err) + } + } + if b, err := decodeRaw(data); err == nil { + check(b) + } + for _, mk := range []func() *Bundle{func() *Bundle { return goodBundle(t) }, func() *Bundle { return goodNZBBundle(t) }} { + b := mk() + mutateBundle(b, sel, v) + check(b) + } + }) +} diff --git a/internal/peernet/bundle/gap_test.go b/internal/peernet/bundle/gap_test.go new file mode 100644 index 000000000..9d2ff2c9a --- /dev/null +++ b/internal/peernet/bundle/gap_test.go @@ -0,0 +1,39 @@ +package bundle + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kipsilabs/altmount/internal/holes" +) + +// A shareable file has no known holes: a gap reference is refused wherever a +// segment table appears. +func TestValidateRejectsGapReferences(t *testing.T) { + cases := map[string]func(b *Bundle){ + "inline": func(b *Bundle) { fileNamed(t, b, "movie.mkv").Segments[1].Ref = GapRef }, + "outer": func(b *Bundle) { fileNamed(t, b, "disc.m2ts").Outer[0].Segments[1].Ref = GapRef }, + "par2": func(b *Bundle) { fileNamed(t, b, "movie.mkv").Par2[0].Segments[0].Ref = GapRef }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + b := clone(goodBundle(t)) + mutate(b) + err := Validate(b) + require.Error(t, err, "a gap reference must be rejected") + require.ErrorIs(t, err, ErrInvalidField) + }) + } +} + +func TestFromLocalRefusesPlaceholderSegments(t *testing.T) { + in := releaseInput(testSpecs()) + in.Files[0].Meta.SegmentData[1].Id = holes.PlaceholderID(2, "salt") + _, err := FromLocal(in) + require.Error(t, err) + if !errors.Is(err, ErrNotShareable) { + t.Fatalf("want a not-shareable error, got %v", err) + } +} diff --git a/internal/peernet/bundle/helpers_test.go b/internal/peernet/bundle/helpers_test.go new file mode 100644 index 000000000..af32db58a --- /dev/null +++ b/internal/peernet/bundle/helpers_test.go @@ -0,0 +1,382 @@ +package bundle + +import ( + "bytes" + "fmt" + "math/rand" + "sort" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/holes" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +// specSeg is a segment of a test file. An empty id is a gap placeholder. +type specSeg struct { + id string + size, start, end int64 +} + +type specOuter struct { + segs []specSeg + key []byte + iv []byte + vol int64 +} + +type specExtent struct { + outer int + off, length int64 +} + +type specPar2 struct { + name string + size int64 + segs []specSeg +} + +// specFile is the test-side description of one local file; meta() renders it +// as the on-disk metadata message independently of the package under test. +type specFile struct { + name string + size int64 + date int64 + segs []specSeg + key, iv []byte + outer []specOuter + extents []specExtent + clips []Clip + par2 []specPar2 +} + +func full(id string, size int64) specSeg { return specSeg{id: id, size: size, start: 0, end: size - 1} } + +func segData(segs []specSeg, salt string) []*metapb.SegmentData { + out := make([]*metapb.SegmentData, len(segs)) + for i, s := range segs { + id := s.id + if id == "" { + id = holes.PlaceholderID(i+1, salt) + } + out[i] = &metapb.SegmentData{Id: id, SegmentSize: s.size, StartOffset: s.start, EndOffset: s.end} + } + return out +} + +func (f specFile) meta() *metapb.FileMetadata { + m := &metapb.FileMetadata{ + FileSize: f.size, + SourceNzbPath: "/local/store/path.nzbz", + StoreRef: "", + Status: metapb.FileStatus_FILE_STATUS_HEALTHY, + CreatedAt: 1700000000, + ModifiedAt: 1700000001, + ReleaseDate: f.date, + OriginalFilename: f.name, + NzbdavId: "dl-id-123", + SegmentData: segData(f.segs, f.name), + } + if len(f.key) > 0 { + m.Encryption = metapb.Encryption_AES + m.AesKey, m.AesIv = f.key, f.iv + } + for _, o := range f.outer { + m.SharedOuterSources = append(m.SharedOuterSources, &metapb.NestedSegmentSource{ + Segments: segData(o.segs, f.name+"o"), AesKey: o.key, AesIv: o.iv, InnerVolumeSize: o.vol, + }) + } + for _, e := range f.extents { + m.NestedSources = append(m.NestedSources, &metapb.NestedSegmentSource{ + SharedOuterSourceIndex: int32(e.outer + 1), InnerOffset: e.off, InnerLength: e.length, + }) + } + for _, c := range f.clips { + m.ClipBoundaries = append(m.ClipBoundaries, &metapb.ClipBoundary{ByteLen: c.ByteLen, Delta_90K: c.Delta90k}) + } + for _, p := range f.par2 { + m.Par2Files = append(m.Par2Files, &metapb.Par2FileReference{ + Filename: p.name, FileSize: p.size, SegmentData: segData(p.segs, p.name), + }) + } + m.KnownHoles = []*metapb.HoleRun{{StartSegment: 1, Count: 1, RecordedAt: 1700000000}} + m.HoleProviderFingerprint = "provider-fingerprint-secret" + return m +} + +func (f specFile) input() FileInput { + return FileInput{Name: f.name, Meta: f.meta(), PasswordFromNZB: true} +} + +func key32() []byte { return bytes.Repeat([]byte{0x42}, 32) } +func iv16() []byte { return bytes.Repeat([]byte{0x24}, 16) } + +// testSpecs returns four files covering every layout the schema holds. +func testSpecs() []specFile { + return []specFile{ + { + name: "movie.mkv", size: 2500, date: 1700000100, + segs: []specSeg{full("m1@x.test", 1000), full("m2@x.test", 1000), full("m3@x.test", 500)}, + par2: []specPar2{{name: "movie.par2", size: 600, segs: []specSeg{full("p1@x.test", 600)}}}, + }, + { + // AES: 100 plain bytes -> 112 encrypted, final block padded. + name: "secret.bin", size: 100, date: 1700000200, + segs: []specSeg{{id: "e1@x.test", size: 200, start: 10, end: 121}}, + key: key32(), iv: iv16(), + }, + { + // Encrypted outer volume with two extents (shared outer source). + name: "nested.mkv", size: 500, date: 1700000300, + outer: []specOuter{{ + segs: []specSeg{full("o1@x.test", 504), full("o2@x.test", 504)}, + key: key32(), iv: iv16(), vol: 1000, + }}, + extents: []specExtent{{0, 10, 300}, {0, 400, 200}}, + }, + { + // Unencrypted outer with clips and partial edges. + name: "disc.m2ts", size: 576, date: 1700000400, + outer: []specOuter{{segs: []specSeg{ + {id: "c1@x.test", size: 500, start: 100, end: 499}, + full("c2@x.test", 500), + full("c3@x.test", 500), + }}}, + extents: []specExtent{{0, 50, 192}, {0, 300, 384}}, + clips: []Clip{{ByteLen: 192, Delta90k: 5}, {ByteLen: 384, Delta90k: -7}}, + }, + } +} + +// testFileID computes the File ID with contentid directly from the spec. +func testFileID(t testing.TB, f specFile) contentid.ID { + t.Helper() + m := f.meta() + if err := expandShared(m); err != nil { + t.Fatal(err) + } + id, err := contentid.FileID(m) + if err != nil { + t.Fatal(err) + } + return id +} + +func expandShared(m *metapb.FileMetadata) error { + for _, ns := range m.NestedSources { + if ns.SharedOuterSourceIndex == 0 { + continue + } + sh := m.SharedOuterSources[ns.SharedOuterSourceIndex-1] + ns.Segments, ns.AesKey, ns.AesIv, ns.InnerVolumeSize = sh.Segments, sh.AesKey, sh.AesIv, sh.InnerVolumeSize + } + return nil +} + +func testFilesRelease(ids []contentid.ID) contentid.ID { + id, err := contentid.ReleaseIDFromFiles(ids) + if err != nil { + panic(err) + } + return id +} + +// assemble builds the expected bundle for specs under the "files" scheme, +// independently of FromLocal. +func assemble(t testing.TB, specs []specFile) *Bundle { + t.Helper() + type pair struct { + id contentid.ID + f specFile + } + ps := make([]pair, len(specs)) + for i, s := range specs { + ps[i] = pair{testFileID(t, s), s} + } + sort.Slice(ps, func(i, j int) bool { return bytes.Compare(ps[i].id[:], ps[j].id[:]) < 0 }) + + b := &Bundle{Scheme: SchemeFiles, Title: "Some Release 2024", PostedUnix: 1700000000} + idx := map[string]int64{} + ref := func(segs []specSeg) []Segment { + if len(segs) == 0 { + return nil + } + out := make([]Segment, len(segs)) + for i, s := range segs { + r := GapRef + if s.id != "" { + v, ok := idx[s.id] + if !ok { + v = int64(len(b.Pool)) + idx[s.id] = v + b.Pool = append(b.Pool, s.id) + } + r = v + } + out[i] = Segment{Ref: r, Size: s.size, Start: s.start, End: s.end} + } + return out + } + var ids []contentid.ID + for _, p := range ps { + f := File{Name: p.f.name, Size: p.f.size, ReleaseDate: p.f.date, FileID: p.id, + Segments: ref(p.f.segs), AESKey: p.f.key, AESIV: p.f.iv, Clips: p.f.clips} + for _, o := range p.f.outer { + f.Outer = append(f.Outer, OuterSource{Segments: ref(o.segs), AESKey: o.key, AESIV: o.iv, VolumeSize: o.vol}) + } + for _, e := range p.f.extents { + f.Extents = append(f.Extents, Extent{Outer: e.outer, InnerOffset: e.off, InnerLength: e.length}) + } + for _, q := range p.f.par2 { + f.Par2 = append(f.Par2, Par2File{Name: q.name, Size: q.size, Segments: ref(q.segs)}) + } + b.Files = append(b.Files, f) + b.TotalSize += f.Size + ids = append(ids, p.id) + } + b.ReleaseID = testFilesRelease(ids) + return b +} + +func goodBundle(t testing.TB) *Bundle { t.Helper(); return assemble(t, testSpecs()) } + +// clone deep-copies a bundle so a test can break one field. +func clone(b *Bundle) *Bundle { + c := *b + c.ReleaseFiles = append([]int(nil), b.ReleaseFiles...) + c.Pool = append([]string(nil), b.Pool...) + c.Files = make([]File, len(b.Files)) + for i, f := range b.Files { + f.Segments = append([]Segment(nil), f.Segments...) + f.AESKey = append([]byte(nil), f.AESKey...) + f.AESIV = append([]byte(nil), f.AESIV...) + f.Extents = append([]Extent(nil), f.Extents...) + f.Clips = append([]Clip(nil), f.Clips...) + f.Outer = append([]OuterSource(nil), f.Outer...) + for j, o := range f.Outer { + o.Segments = append([]Segment(nil), o.Segments...) + o.AESKey = append([]byte(nil), o.AESKey...) + o.AESIV = append([]byte(nil), o.AESIV...) + f.Outer[j] = o + } + f.Par2 = append([]Par2File(nil), f.Par2...) + for j, p := range f.Par2 { + p.Segments = append([]Segment(nil), p.Segments...) + f.Par2[j] = p + } + c.Files[i] = f + } + return &c +} + +func fileNamed(t testing.TB, b *Bundle, name string) *File { + t.Helper() + for i := range b.Files { + if b.Files[i].Name == name { + return &b.Files[i] + } + } + t.Fatalf("no file %q", name) + return nil +} + +// synthetic release for size measurements: n segments of random ids. +func syntheticIDs(n int, seed int64) []string { + r := rand.New(rand.NewSource(seed)) + const alpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + domains := []string{"powerpost2000AA.local", "nyuu.local", "JBinUp.local", "news.example"} + out := make([]string, n) + for i := range out { + l := 20 + r.Intn(12) + b := make([]byte, l) + for j := range b { + b[j] = alpha[r.Intn(len(alpha))] + } + out[i] = fmt.Sprintf("%s@%s", b, domains[r.Intn(len(domains))]) + } + return out +} + +// syntheticBundle is one plain file of n full segments (files scheme). +func syntheticBundle(t testing.TB, n int, segSize int64) *Bundle { + t.Helper() + ids := syntheticIDs(n, 1) + segs := make([]specSeg, n) + for i := range segs { + segs[i] = full(ids[i], segSize) + } + last := n - 1 + segs[last] = full(ids[last], segSize/2) + return assemble(t, []specFile{{name: "big.mkv", size: segSize*int64(n-1) + segSize/2, date: 1700000000, segs: segs}}) +} + +var testNow = time.Unix(1_700_001_000, 0) + +// nzbLayout is the NZB file list the nzb-scheme fixture is built on: the +// imported files' segments, plus one NZB file that was not imported. +func nzbLayout() [][]string { + return [][]string{ + {"m1@x.test", "m2@x.test", "m3@x.test"}, + {"p1@x.test"}, + {"e1@x.test"}, + {"o1@x.test", "o2@x.test"}, + {"c1@x.test", "c2@x.test", "c3@x.test"}, + {"x1@x.test", "x2@x.test"}, + } +} + +func nzbStore(files [][]string) *metapb.NzbStore { + st := &metapb.NzbStore{} + for _, f := range files { + fe := &metapb.NzbFileEntry{} + for i, id := range f { + fe.Segments = append(fe.Segments, &metapb.NzbSeg{Id: id, Number: int32(i + 1), Bytes: 1}) + } + st.Files = append(st.Files, fe) + } + return st +} + +// assembleNZB builds the expected nzb-scheme bundle independently of FromLocal. +func assembleNZB(t testing.TB, specs []specFile, files [][]string) *Bundle { + t.Helper() + b := assemble(t, specs) + b.Scheme = SchemeNZB + b.Pool = nil + idx := map[string]int64{} + for _, f := range files { + b.ReleaseFiles = append(b.ReleaseFiles, len(f)) + for _, id := range f { + if _, ok := idx[id]; !ok { + idx[id] = int64(len(b.Pool)) + } + b.Pool = append(b.Pool, id) + } + } + remap := func(segs []Segment, old []string) { + for i := range segs { + if segs[i].Ref != GapRef { + segs[i].Ref = idx[old[segs[i].Ref]] + } + } + } + old := assemble(t, specs).Pool + for i := range b.Files { + remap(b.Files[i].Segments, old) + for j := range b.Files[i].Outer { + remap(b.Files[i].Outer[j].Segments, old) + } + for j := range b.Files[i].Par2 { + remap(b.Files[i].Par2[j].Segments, old) + } + } + rid, err := contentid.ReleaseID(nzbStore(files)) + if err != nil { + t.Fatal(err) + } + b.ReleaseID = rid + return b +} + +func goodNZBBundle(t testing.TB) *Bundle { t.Helper(); return assembleNZB(t, testSpecs(), nzbLayout()) } diff --git a/internal/peernet/bundle/limits.go b/internal/peernet/bundle/limits.go new file mode 100644 index 000000000..873771dc1 --- /dev/null +++ b/internal/peernet/bundle/limits.go @@ -0,0 +1,100 @@ +package bundle + +import "errors" + +// Limits. Several are design calls the owner may tune; the hard caps named in +// the design (8 MB compressed, 64 MB decompressed, 500k segments, 10k files) +// are enforced while streaming and again on the decoded structure. +const ( + // SchemaVersion is the only major version this package reads and writes. + SchemaVersion = 1 + + // MaxCompressedSize bounds the bundle as it travels on the wire. + MaxCompressedSize = 8 << 20 + // MaxDecompressedSize bounds the canonical (uncompressed) bytes. The + // decompressor stops as soon as the output would exceed it. + MaxDecompressedSize = 64 << 20 + // MaxWindowSize is the largest zstd window a frame may ask for. The + // builder uses the same value, so honest bundles always fit. + MaxWindowSize = 8 << 20 + + // MaxSegments caps the segment positions of a bundle, counted over every + // segment table (main files, shared outer sources and PAR2 files), and + // separately the message-ID pool. + MaxSegments = 500_000 + // MaxFiles caps the files of one bundle. + MaxFiles = 10_000 + // MaxReleaseFiles caps the NZB file entries of the "nzb" scheme release + // list (it includes files that were not imported). + MaxReleaseFiles = 50_000 + // MaxMessageIDLen caps a canonical message-ID (no angle brackets). RFC + // 5536 limits message-IDs to 250 octets including the brackets. + MaxMessageIDLen = 248 + // MaxFilenameLen caps a sanitised file name in bytes. + MaxFilenameLen = 255 + // MaxTitleLen caps the release title in bytes (same as release records). + MaxTitleLen = 256 + // MaxPasswordLen caps the opaque NZB password in bytes. + MaxPasswordLen = 256 + // MaxNestingDepth is the deepest archive nesting a file may describe: a + // file read through its outer archive volumes. The schema cannot express + // deeper nesting or a cycle, so the cap is structural. + MaxNestingDepth = 1 + // MaxOuterSources caps the shared outer sources of one file. + MaxOuterSources = 10_000 + // MaxExtents caps the nested extents of the whole bundle. + MaxExtents = 100_000 + // MaxClips caps the clip boundaries of one file. + MaxClips = 100_000 + // MaxTotalClips, MaxTotalOuterSources and MaxTotalPar2Files are the + // bundle-wide budgets next to the segment and extent budgets, so many + // files cannot each spend their own per-file allowance. + MaxTotalClips = 200_000 + MaxTotalOuterSources = 50_000 + MaxTotalPar2Files = 10_000 + // MaxPar2Files caps the PAR2 file references of one file. + MaxPar2Files = 1_000 + + // MaxSegmentSize is the largest decoded size of one article: real posts + // are 0.25-1 MB, a few go to a few MB. + MaxSegmentSize = 8 << 20 + // MaxFileSize caps one file and the bundle total (1 PiB, as records). + MaxFileSize = int64(1) << 50 + // MaxClipDelta bounds a clip's 90 kHz offset (about 10 years). + MaxClipDelta = int64(90_000) * 86_400 * 3_650 + // ClipPacketSize is the BDAV source packet size every clip length is a + // multiple of. + ClipPacketSize = 192 + + maxUnixSeconds = 4102444800 // 2100-01-01 + idSize = 32 + aesBlockSize = 16 +) + +// Sentinel errors. Callers can count rejection reasons with errors.Is. +var ( + ErrTooLarge = errors.New("bundle: too large") + ErrMalformed = errors.New("bundle: malformed or non-canonical encoding") + ErrUnsupportedVersion = errors.New("bundle: unsupported schema version") + ErrUnknownScheme = errors.New("bundle: unknown release id scheme") + ErrInvalidField = errors.New("bundle: invalid field value") + ErrBadFilename = errors.New("bundle: invalid file name") + ErrDuplicateName = errors.New("bundle: duplicate file name") + ErrBadMessageID = errors.New("bundle: invalid message-id") + ErrLimit = errors.New("bundle: count limit exceeded") + ErrOutOfRange = errors.New("bundle: reference or offset out of range") + ErrSizeMismatch = errors.New("bundle: extents do not add up to the declared size") + ErrImplausible = errors.New("bundle: implausible size for the segment count") + ErrBadEncryption = errors.New("bundle: invalid encryption material") + ErrBadClip = errors.New("bundle: invalid clip boundaries") + ErrBadPool = errors.New("bundle: invalid message-id pool") + ErrOverlap = errors.New("bundle: overlapping extents") + ErrBundleIDMismatch = errors.New("bundle: bundle id does not match the announced id") + ErrReleaseIDMismatch = errors.New("bundle: release id does not match the announced id") + ErrFileIDMismatch = errors.New("bundle: file id does not match its segments") + ErrDuplicateFile = errors.New("bundle: duplicate file id") + ErrNotShareable = errors.New("bundle: release is not shareable") + ErrUnresolved = errors.New("bundle: metadata segments are not resolved") + ErrCoverage = errors.New("bundle: segments are not exactly one NZB file entry") + ErrSegmentNotInNZB = errors.New("bundle: file segment is not in the release segment list") +) diff --git a/internal/peernet/bundle/sanitize.go b/internal/peernet/bundle/sanitize.go new file mode 100644 index 000000000..03f536b60 --- /dev/null +++ b/internal/peernet/bundle/sanitize.go @@ -0,0 +1,170 @@ +package bundle + +import ( + "fmt" + "strings" + "unicode" + "unicode/utf8" + + "github.com/kipsilabs/altmount/internal/holes" +) + +// unnamedFile replaces a file name that sanitises to nothing ("", ".", ".."). +const unnamedFile = "unnamed" + +// maxExtLen is the longest extension kept when a long name is shortened. +const maxExtLen = 16 + +// disallowedRune reports runes that must never appear in peer-supplied text: +// control characters (including NUL), path separators, bidirectional +// override and isolate characters, and line or paragraph separators. It is +// the same set the release records use. +func disallowedRune(r rune) bool { + switch { + case unicode.IsControl(r), r == '/', r == '\\': + return true + case r >= 0x202A && r <= 0x202E, r >= 0x2066 && r <= 0x2069: + return true + case r == 0x200E, r == 0x200F, r == 0x061C, r == 0x2028, r == 0x2029, r == 0xFEFF: + return true + } + return false +} + +// cutBytes shortens s to at most max bytes on a rune boundary. +func cutBytes(s string, max int) string { + if len(s) <= max { + return s + } + s = s[:max] + for !utf8.ValidString(s) { + s = s[:len(s)-1] + } + return s +} + +// cleanText repairs local input: invalid UTF-8 is dropped, disallowed runes +// become spaces, whitespace is collapsed and the result is cut to max bytes. +// It is for the builder only; decoders reject instead. +func cleanText(s string, max int) string { + s = strings.ToValidUTF8(s, "") + s = strings.Map(func(r rune) rune { + if disallowedRune(r) { + return ' ' + } + return r + }, s) + s = strings.Join(strings.Fields(s), " ") + return strings.TrimSpace(cutBytes(s, max)) +} + +// SanitizeTitle repairs a release title (builder side). +func SanitizeTitle(s string) string { return cleanText(s, MaxTitleLen) } + +func checkTitle(s string) error { + switch { + case s == "": + return fmt.Errorf("%w: title is empty", ErrInvalidField) + case len(s) > MaxTitleLen: + return fmt.Errorf("%w: title too long", ErrInvalidField) + case !utf8.ValidString(s): + return fmt.Errorf("%w: title is not valid UTF-8", ErrInvalidField) + } + for _, r := range s { + if disallowedRune(r) { + return fmt.Errorf("%w: title contains a forbidden character", ErrInvalidField) + } + } + return nil +} + +// reservedDevice reports Windows device names, with or without extension. +func reservedDevice(name string) bool { + stem := name + if i := strings.IndexByte(stem, '.'); i >= 0 { + stem = stem[:i] + } + stem = strings.ToUpper(strings.TrimRight(stem, " ")) + switch stem { + case "CON", "PRN", "AUX", "NUL": + return true + } + if len(stem) >= 4 && (strings.HasPrefix(stem, "COM") || strings.HasPrefix(stem, "LPT")) { + switch stem[3:] { + case "1", "2", "3", "4", "5", "6", "7", "8", "9", "¹", "²", "³": + return true + } + } + return false +} + +// shortenName cuts name to MaxFilenameLen bytes, keeping a short extension. +func shortenName(name string) string { + if len(name) <= MaxFilenameLen { + return name + } + ext := "" + if i := strings.LastIndexByte(name, '.'); i > 0 && len(name)-i <= maxExtLen { + ext = name[i:] + } + stem := cutBytes(name[:len(name)-len(ext)], MaxFilenameLen-len(ext)) + return strings.TrimRight(stem, ". ") + ext +} + +// SanitizeFilename turns an untrusted file name into a safe base name: only +// the part after the last path separator is kept; control, bidirectional and +// separator characters become spaces; trailing dots and spaces are removed; +// ".", ".." and empty names become "unnamed"; Windows device names get a "_" +// prefix; the result is at most MaxFilenameLen bytes. The function is +// idempotent, and checkFilename accepts exactly its fixed points. +func SanitizeFilename(s string) string { + s = strings.ToValidUTF8(s, "") + if i := strings.LastIndexAny(s, `/\`); i >= 0 { + s = s[i+1:] + } + s = strings.Map(func(r rune) rune { + if disallowedRune(r) { + return ' ' + } + return r + }, s) + s = strings.TrimRight(strings.Join(strings.Fields(s), " "), ". ") + if s == "" { + return unnamedFile + } + s = shortenName(s) + if reservedDevice(s) { + s = shortenName("_" + s) + } + return s +} + +func checkFilename(s string) error { + if s == "" || len(s) > MaxFilenameLen || !utf8.ValidString(s) || SanitizeFilename(s) != s { + return fmt.Errorf("%w: %q", ErrBadFilename, truncateForError(s)) + } + return nil +} + +func truncateForError(s string) string { return cutBytes(strings.ToValidUTF8(s, "?"), 40) } + +// CanonicalMessageID strips one pair of angle brackets and validates the +// result: 1 to MaxMessageIDLen printable ASCII characters without brackets, +// and not a gap placeholder id. Message-IDs reach NNTP commands, so anything +// that could end a command line is refused. +func CanonicalMessageID(s string) (string, error) { + s = strings.TrimPrefix(s, "<") + s = strings.TrimSuffix(s, ">") + if s == "" || len(s) > MaxMessageIDLen { + return "", fmt.Errorf("%w: length %d", ErrBadMessageID, len(s)) + } + for i := 0; i < len(s); i++ { + if c := s[i]; c <= 0x20 || c >= 0x7f || c == '<' || c == '>' { + return "", fmt.Errorf("%w: forbidden character", ErrBadMessageID) + } + } + if holes.IsPlaceholderID(s) { + return "", fmt.Errorf("%w: reserved placeholder prefix", ErrBadMessageID) + } + return s, nil +} diff --git a/internal/peernet/bundle/sanitize_test.go b/internal/peernet/bundle/sanitize_test.go new file mode 100644 index 000000000..846c8a191 --- /dev/null +++ b/internal/peernet/bundle/sanitize_test.go @@ -0,0 +1,127 @@ +package bundle + +import ( + "errors" + "strings" + "testing" + "unicode/utf8" +) + +func TestSanitizeFilename(t *testing.T) { + long := strings.Repeat("a", 400) + ".mkv" + tests := []struct { + name, in, want string + }{ + {"plain", "movie.mkv", "movie.mkv"}, + {"unix traversal", "../x", "x"}, + {"deep traversal", "../../etc/passwd", "passwd"}, + {"subdir", "a/b", "b"}, + {"backslash", `a\b`, "b"}, + {"mixed separators", `a/b\c.mkv`, "c.mkv"}, + {"windows path", `C:\Users\x\file.mkv`, "file.mkv"}, + {"nul inside", "a\x00b.mkv", "a b.mkv"}, + {"nul traversal", "..\x00/x", "x"}, + {"dot", ".", "unnamed"}, + {"dotdot", "..", "unnamed"}, + {"dotdot slash", "../", "unnamed"}, + {"empty", "", "unnamed"}, + {"only spaces", " ", "unnamed"}, + {"trailing dots", "name...", "name"}, + {"trailing space and dot", "name. .", "name"}, + {"leading dotdot name stays a name", "..x", "..x"}, + {"bidi override", "a\u202Eevil.mkv", "a evil.mkv"}, + {"bidi isolate", "a\u2066b", "a b"}, + {"line separator", "a\u2028b", "a b"}, + {"bom", "\uFEFFa", "a"}, + {"control", "a\tb\nc", "a b c"}, + {"invalid utf8", "a\xffb", "ab"}, + {"collapse whitespace", "a b", "a b"}, + {"reserved CON", "CON", "_CON"}, + {"reserved con ext", "con.txt", "_con.txt"}, + {"reserved COM1", "COM1.mkv", "_COM1.mkv"}, + {"reserved LPT9", "lpt9", "_lpt9"}, + {"reserved NUL", "NUL", "_NUL"}, + {"not reserved", "console.mkv", "console.mkv"}, + {"reserved superscript", "COM\u00B9", "_COM\u00B9"}, + {"long", long, strings.Repeat("a", 251) + ".mkv"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := SanitizeFilename(tc.in) + if got != tc.want { + t.Fatalf("SanitizeFilename(%q) = %q, want %q", tc.in, got, tc.want) + } + if again := SanitizeFilename(got); again != got { + t.Fatalf("not idempotent: %q -> %q", got, again) + } + if err := checkFilename(got); err != nil { + t.Fatalf("sanitised name rejected: %v", err) + } + if len(got) > MaxFilenameLen || !utf8.ValidString(got) { + t.Fatalf("bad length or encoding: %q", got) + } + }) + } +} + +func TestSanitizeFilenameLongMultibyteCutOnRune(t *testing.T) { + in := strings.Repeat("é", 300) + got := SanitizeFilename(in) + if len(got) > MaxFilenameLen || !utf8.ValidString(got) || got == "" { + t.Fatalf("got %d bytes valid=%v", len(got), utf8.ValidString(got)) + } + if got != strings.Repeat("é", 127) { + t.Fatalf("got %q", got) + } +} + +func TestCheckFilenameRejects(t *testing.T) { + for _, in := range []string{"", ".", "..", "a/b", `a\b`, "a\x00b", "../x", " a", "a ", "a.", "CON", "con.txt", + "a\u202Eb", "a b", "a\xffb", strings.Repeat("a", 256)} { + if err := checkFilename(in); !errors.Is(err, ErrBadFilename) { + t.Errorf("checkFilename(%q) = %v, want ErrBadFilename", in, err) + } + } + if err := checkFilename("ok name.mkv"); err != nil { + t.Fatal(err) + } +} + +func TestSanitizeTitle(t *testing.T) { + if got := SanitizeTitle(" A\u202E b/c\\d\x00 "); got != "A b c d" { + t.Fatalf("got %q", got) + } + if got := SanitizeTitle(strings.Repeat("x", 1000)); len(got) != MaxTitleLen { + t.Fatalf("len %d", len(got)) + } + if got := SanitizeTitle(""); got != "" { + t.Fatalf("got %q", got) + } + if err := checkTitle("a\u202Eb"); !errors.Is(err, ErrInvalidField) { + t.Fatalf("err %v", err) + } + if err := checkTitle(""); !errors.Is(err, ErrInvalidField) { + t.Fatalf("empty title: %v", err) + } +} + +func TestCanonicalMessageID(t *testing.T) { + ok := map[string]string{ + "abc@x.test": "abc@x.test", + "": "abc@x.test", + "a$b%c@x.test": "a$b%c@x.test", + } + for in, want := range ok { + got, err := CanonicalMessageID(in) + if err != nil || got != want { + t.Errorf("%q -> %q, %v", in, got, err) + } + } + bad := []string{"", "<>", "a b@x", "a\r\nQUIT@x", "a\x00@x", "a\x7f@x", "é@x", "ab@x", + "<>", strings.Repeat("a", MaxMessageIDLen+1), "altmount-gap-1-abc", ""} + for _, in := range bad { + if _, err := CanonicalMessageID(in); !errors.Is(err, ErrBadMessageID) { + t.Errorf("%q: %v, want ErrBadMessageID", in, err) + } + } +} diff --git a/internal/peernet/bundle/types.go b/internal/peernet/bundle/types.go new file mode 100644 index 000000000..bb65d48c0 --- /dev/null +++ b/internal/peernet/bundle/types.go @@ -0,0 +1,122 @@ +package bundle + +import ( + "encoding/hex" + + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +// Scheme says how the Release ID of a bundle is defined. +type Scheme uint8 + +const ( + // SchemeNZB: the Release ID is the contentid Release ID over the release's + // full ordered segment list. The bundle carries that list, including NZB + // files that were not imported, so the receiver can recompute it. + SchemeNZB Scheme = 1 + // SchemeFiles: the Release ID is SHA-256 over the tag + // "altmount-peernet/release-files/v1" followed by the sorted File IDs. + SchemeFiles Scheme = 2 +) + +func (s Scheme) String() string { + switch s { + case SchemeNZB: + return "nzb" + case SchemeFiles: + return "files" + } + return "unknown" +} + +// BundleID is the SHA-256 of the canonical uncompressed bundle bytes. +type BundleID [idSize]byte + +// String returns the id as 64 lower-case hex characters. +func (id BundleID) String() string { return hex.EncodeToString(id[:]) } + +// GapRef is the Segment.Ref of a synthetic gap placeholder (an article the +// NZB never listed). A gap has no message-ID; the receiver derives a local +// placeholder id. +const GapRef int64 = -1 + +// Segment is one entry of a segment table: the article it reads (an index +// into Bundle.Pool, or GapRef), its decoded size, and the usable byte range +// inside it (inclusive offsets, as on disk). +type Segment struct { + Ref int64 + Size int64 + Start int64 + End int64 +} + +// OuterSource is one outer archive volume a nested file reads through. With +// a key the volume is AES-CBC encrypted and VolumeSize is its decrypted size. +type OuterSource struct { + Segments []Segment + AESKey []byte + AESIV []byte + VolumeSize int64 +} + +// Extent is one slice of an outer source that holds file bytes. Extents are +// concatenated in order to form the file. +type Extent struct { + Outer int + InnerOffset int64 + InnerLength int64 +} + +// Clip is one clip of a byte-concatenated multi-clip main feature. +type Clip struct { + ByteLen int64 + Delta90k int64 +} + +// Par2File references one PAR2 repair file of the release. +type Par2File struct { + Name string + Size int64 + Segments []Segment +} + +// File is one virtual file of the release. +type File struct { + // Name is a sanitised base name. It is never a path. + Name string + Size int64 + // ReleaseDate is the Unix time of the original post (0 = unknown). + ReleaseDate int64 + // FileID is the contentid File ID; Validate-time checks recompute it. + FileID contentid.ID + // Segments is the file's own segment table (empty for nested files). + Segments []Segment + // AESKey and AESIV are set together for a directly AES-encrypted file. + AESKey []byte + AESIV []byte + // Outer and Extents describe a file read through outer archive volumes. + Outer []OuterSource + Extents []Extent + Clips []Clip + Par2 []Par2File +} + +// Bundle is the decoded, structurally valid form of a release bundle. +type Bundle struct { + Scheme Scheme + ReleaseID contentid.ID + Title string + // PostedUnix is the release post date (Unix seconds). + PostedUnix int64 + // TotalSize is the sum of the file sizes. + TotalSize int64 + // NZBPassword is an opaque string taken from the NZB; empty if none. + NZBPassword string + // ReleaseFiles (SchemeNZB only) lists, per NZB file in NZB order, how many + // real segment ids it holds. The ids are Pool in order. + ReleaseFiles []int + // Pool holds canonical message-IDs. Segment.Ref indexes it. + Pool []string + // Files are sorted by FileID ascending. + Files []File +} diff --git a/internal/peernet/bundle/validate.go b/internal/peernet/bundle/validate.go new file mode 100644 index 000000000..615d11932 --- /dev/null +++ b/internal/peernet/bundle/validate.go @@ -0,0 +1,577 @@ +package bundle + +import ( + "fmt" + "sort" + "strings" + "unicode" + "unicode/utf8" +) + +// Reader assumptions and the checks that uphold them. +// +// Everything a peer sends is hostile, so a bundle must never be able to break +// an assumption the readers of the on-disk metadata make. Line numbers refer +// to the readers as of this writing. +// +// reader assumption reader code check here +// -------------------------------------------------------- -------------------------------------------- ------------------------------- +// usable bytes of a segment are End-Start+1 and positive; nzbfilesystem/metadata_remote_file.go:1176, checkTable: 0 <= Start <= End +// a segment with usable <= 0 is silently dropped, shifting usenet/segment.go:112-114 <= Size-1 +// every later offset +// a gap or patch is zero-filled with End+1 bytes, so End usenet/usenet_reader.go:947-949,1057-1076 checkTable: Size <= MaxSegmentSize +// bounds an allocation (8 MiB), End < Size +// a placeholder message-id prefix means "zero-fill, do not usenet/usenet_reader.go:1057, CanonicalMessageID rejects the +// fetch"; a shareable file has no gaps at all importer/validation/fast_fail.go:282 prefix; Ref == GapRef is refused +// a message-id is sent in NNTP commands; a space or CRLF usenet/usenet_reader.go (BODY/STAT by id) CanonicalMessageID: printable +// would split or inject a command ASCII, no brackets, <= 248 bytes +// FileSize >= 1 (a zero size is an invalid range, never a nzbfilesystem/metadata_remote_file.go:2128 validateFile: 1 <= Size <= cap +// readable file) +// unencrypted file: segment coverage equals FileSize metadata_remote_file.go:2236-2246 (reads are validateFile: coverage == Size +// (coverage past EOF is never requested; a shortfall is a bounded by coverage, EOF by FileSize), +// truncated tail) 1140-1155 (truncatedTailError) +// AES file: coverage holds the padded final block, i.e. metadata_remote_file.go:2231-2234, validateFile: padded(Size) <= +// between the block-aligned size and FileSize+15 encryption/aes/aes_cipher.go:74, coverage <= Size+15 +// aes_decrypt_reader.go:82-90 +// AES key is 16/24/32 bytes, IV 16 bytes, both present encryption/aes/aes_decrypt_reader.go:42-48, checkKeyIV +// together metadata_remote_file.go:1751-1756 +// rclone and header encryption need local secrets the metadata_remote_file.go:1721-1745 not representable; the builder +// bundle never carries refuses such files (not shareable) +// nested reads take priority over inline segments, so a metadata_remote_file.go:1670-1676 validateFile: nested files have +// nested file with both is ambiguous no inline segments, no file key +// extents are concatenated by InnerLength, so they must metadata_remote_file.go:2321-2357 checkExtents: sum == Size, each +// add up to FileSize; a zero or negative length is skipped length >= 1 +// an extent reads outer bytes [InnerOffset, +InnerLength) metadata_remote_file.go:2369-2393 checkExtents: inside the encrypted +// of the (decrypted) outer volume volume size, or inside the plain +// outer coverage +// an encrypted outer volume is read as ciphertext from 0 metadata_remote_file.go:2371-2393, checkOuter: padded(VolumeSize) <= +// to padded(InnerVolumeSize)-1 and needs InnerVolumeSize aes_decrypt_reader.go:82-90 coverage <= VolumeSize+15 +// a shared outer source index must exist metadata/expand.go:29-37, Extent.Outer in range +// metadata/store.go:280-300 (resolveRefs) +// every shared outer source is used (else it is dead data) metadata/service.go (v3 write dissolves them) checkExtents: no unused outer +// clip lengths are positive 192-byte multiples and cover nzbfilesystem/tsremux_reader.go:21-38, checkClips +// the whole file: the remux aligns reads on the packet grid metadata_remote_file.go:1690-1710, +// and clips past FileSize are never produced importer/archive/iso_expansion.go:206-228 +// remux deltas are small enough to add to 90 kHz PTS/DTS nzbfilesystem/tsremux.go checkClips: |delta| <= MaxClipDelta +// without wrapping +// metadata file names are single path components metadata/service.go:196-228 (metaFilePath) checkFilename (sanitiser fixed +// point), unique ignoring case +// hole-fill and hedging are only enabled for plain files nzbfilesystem/holes.go:35-37 nothing to check: derived locally +// arithmetic on sizes never overflows int64 (all readers use int64 sums) every size <= 2^50, counts +// <= 500k * 8 MiB; sums checked +// +// The nesting depth is exactly one by construction: outer sources are plain +// segment tables, so a cycle or deeper nesting cannot be expressed. + +// validator carries the running totals of one Validate call. +type validator struct { + b *Bundle + segs int + extents int + used []bool // pool entries referenced (files scheme) + names map[string]struct{} + // nzb scheme: the pool range of every NZB file entry, and which entries + // are read inline (once, by one file) or as shared outer/PAR2 sources. + entStart []int + entLen []int + inline map[int]struct{} + sharedE map[int]struct{} + total int64 + lastFile *File +} + +// Validate checks every structural rule of the bundle schema: limits, ranges, +// sums, encryption sizes, names and the message-ID pool. It does not touch the +// network or recompute identifiers (Verify does that). It never panics and +// never allocates in proportion to a size the bundle merely claims. +func Validate(b *Bundle) error { + if b == nil { + return fmt.Errorf("%w: nil bundle", ErrInvalidField) + } + if b.Scheme != SchemeNZB && b.Scheme != SchemeFiles { + return fmt.Errorf("%w: %d", ErrUnknownScheme, b.Scheme) + } + if b.ReleaseID.IsZero() { + return fmt.Errorf("%w: zero release id", ErrInvalidField) + } + if err := checkTitle(b.Title); err != nil { + return err + } + if b.PostedUnix < 1 || b.PostedUnix > maxUnixSeconds { + return fmt.Errorf("%w: post date out of range", ErrInvalidField) + } + if err := checkPassword(b.NZBPassword); err != nil { + return err + } + v := &validator{b: b, names: map[string]struct{}{}} + if err := v.checkPool(); err != nil { + return err + } + switch { + case len(b.Files) == 0: + return fmt.Errorf("%w: no files", ErrInvalidField) + case len(b.Files) > MaxFiles: + return fmt.Errorf("%w: %d files", ErrLimit, len(b.Files)) + } + for i := range b.Files { + if err := v.checkFile(&b.Files[i]); err != nil { + return fmt.Errorf("file %d: %w", i, err) + } + } + if b.Scheme == SchemeFiles { + for i, used := range v.used { + if !used { + return fmt.Errorf("%w: pool entry %d is not referenced", ErrBadPool, i) + } + } + } + if v.total != b.TotalSize { + return fmt.Errorf("%w: total size %d, files add up to %d", ErrSizeMismatch, b.TotalSize, v.total) + } + return nil +} + +func checkPassword(p string) error { + if len(p) > MaxPasswordLen || !utf8.ValidString(p) { + return fmt.Errorf("%w: password", ErrInvalidField) + } + for _, r := range p { + if unicode.IsControl(r) { + return fmt.Errorf("%w: password contains a control character", ErrInvalidField) + } + } + return nil +} + +func (v *validator) checkPool() error { + b := v.b + if len(b.Pool) > MaxSegments { + return fmt.Errorf("%w: %d pool entries", ErrLimit, len(b.Pool)) + } + for i, id := range b.Pool { + if c, err := CanonicalMessageID(id); err != nil || c != id { + return fmt.Errorf("%w: pool entry %d", ErrBadMessageID, i) + } + } + if b.Scheme == SchemeNZB { + if len(b.ReleaseFiles) > MaxReleaseFiles { + return fmt.Errorf("%w: %d release files", ErrLimit, len(b.ReleaseFiles)) + } + sum := 0 + for _, n := range b.ReleaseFiles { + if n < 0 || n > MaxSegments { + return fmt.Errorf("%w: release file segment count", ErrBadPool) + } + sum += n + } + if len(b.ReleaseFiles) == 0 || sum != len(b.Pool) { + return fmt.Errorf("%w: release list covers %d of %d pool entries", ErrBadPool, sum, len(b.Pool)) + } + seen := make(map[string]struct{}, len(b.Pool)) + for _, id := range b.Pool { + if _, dup := seen[id]; dup { + return fmt.Errorf("%w: duplicate message-id in the release list", ErrBadPool) + } + seen[id] = struct{}{} + } + pos := 0 + v.entStart, v.entLen = make([]int, len(b.ReleaseFiles)), b.ReleaseFiles + for i, n := range b.ReleaseFiles { + v.entStart[i] = pos + pos += n + } + v.inline, v.sharedE = map[int]struct{}{}, map[int]struct{}{} + return nil + } + if len(b.ReleaseFiles) != 0 { + return fmt.Errorf("%w: release list on a files-scheme bundle", ErrInvalidField) + } + seen := make(map[string]struct{}, len(b.Pool)) + for _, id := range b.Pool { + if _, dup := seen[id]; dup { + return fmt.Errorf("%w: duplicate message-id", ErrBadPool) + } + seen[id] = struct{}{} + } + v.used = make([]bool, len(b.Pool)) + return nil +} + +func validDate(d int64) bool { return d == 0 || (d >= 1 && d <= maxUnixSeconds) } + +func (v *validator) checkFile(f *File) error { + if err := checkFilename(f.Name); err != nil { + return err + } + key := strings.ToLower(f.Name) + if _, dup := v.names[key]; dup { + return fmt.Errorf("%w: %q", ErrDuplicateName, f.Name) + } + v.names[key] = struct{}{} + + if err := v.checkFileOrder(f); err != nil { + return err + } + if len(f.Outer) > MaxOuterSources { + return fmt.Errorf("%w: %d outer sources", ErrLimit, len(f.Outer)) + } + if v.extents += len(f.Extents); v.extents > MaxExtents { + return fmt.Errorf("%w: more than %d extents", ErrLimit, MaxExtents) + } + if len(f.Clips) > MaxClips { + return fmt.Errorf("%w: %d clips", ErrLimit, len(f.Clips)) + } + if len(f.Par2) > MaxPar2Files { + return fmt.Errorf("%w: %d par2 files", ErrLimit, len(f.Par2)) + } + if f.Size < 1 || f.Size > MaxFileSize { + return fmt.Errorf("%w: file size %d", ErrInvalidField, f.Size) + } + if !validDate(f.ReleaseDate) { + return fmt.Errorf("%w: release date", ErrInvalidField) + } + if err := checkKeyIV(f.AESKey, f.AESIV); err != nil { + return err + } + + nested := len(f.Extents) > 0 || len(f.Outer) > 0 + switch { + case nested && len(f.Segments) > 0: + return fmt.Errorf("%w: nested file lists inline segments", ErrInvalidField) + case nested && len(f.AESKey) > 0: + return fmt.Errorf("%w: nested file has a file-level key", ErrBadEncryption) + case nested: + if err := v.checkNested(f); err != nil { + return err + } + default: + if err := v.checkInline(f); err != nil { + return err + } + } + if len(f.Clips) > 0 { + if err := checkClips(f); err != nil { + return err + } + } + if err := v.checkPar2(f); err != nil { + return err + } + if err := v.checkCoverage(f); err != nil { + return err + } + if v.total > MaxFileSize-f.Size { + return fmt.Errorf("%w: bundle total above the cap", ErrInvalidField) + } + v.total += f.Size + return nil +} + +// checkFileOrder requires strictly ascending, non-zero File IDs. +func (v *validator) checkFileOrder(f *File) error { + switch { + case f.FileID.IsZero(): + return fmt.Errorf("%w: zero file id", ErrInvalidField) + case v.lastFile != nil && v.lastFile.FileID == f.FileID: + return fmt.Errorf("%w: %s", ErrDuplicateFile, f.FileID) + case v.lastFile != nil && string(v.lastFile.FileID[:]) > string(f.FileID[:]): + return fmt.Errorf("%w: files are not sorted by file id", ErrInvalidField) + } + v.lastFile = f + return nil +} + +func checkKeyIV(key, iv []byte) error { + if len(key) == 0 && len(iv) == 0 { + return nil + } + if len(key) != 16 && len(key) != 24 && len(key) != 32 { + return fmt.Errorf("%w: key length %d", ErrBadEncryption, len(key)) + } + if len(iv) != aesBlockSize { + return fmt.Errorf("%w: iv length %d", ErrBadEncryption, len(iv)) + } + return nil +} + +// padded is the AES-CBC encrypted size of n plaintext bytes. +func padded(n int64) int64 { + if r := n % aesBlockSize; r != 0 { + return n + aesBlockSize - r + } + return n +} + +// checkInline validates a file read straight from its own segments. +func (v *validator) checkInline(f *File) error { + if len(f.Segments) == 0 { + return fmt.Errorf("%w: file has no data", ErrInvalidField) + } + covered, err := v.checkTable(f.Segments) + if err != nil { + return err + } + if f.Size > int64(len(f.Segments))*MaxSegmentSize { + return fmt.Errorf("%w: %d bytes in %d segments", ErrImplausible, f.Size, len(f.Segments)) + } + if len(f.AESKey) > 0 { + if covered < padded(f.Size) || covered > f.Size+aesBlockSize-1 { + return fmt.Errorf("%w: encrypted file of %d bytes covered by %d", ErrBadEncryption, f.Size, covered) + } + return nil + } + if covered != f.Size { + return fmt.Errorf("%w: size %d, segments cover %d", ErrSizeMismatch, f.Size, covered) + } + return nil +} + +// checkTable validates one segment table and returns the bytes it covers. +func (v *validator) checkTable(segs []Segment) (int64, error) { + if v.segs += len(segs); v.segs > MaxSegments { + return 0, fmt.Errorf("%w: more than %d segments", ErrLimit, MaxSegments) + } + poolLen := int64(len(v.b.Pool)) + var covered int64 + for i, s := range segs { + switch { + case s.Ref == GapRef: + // A shareable file has no known holes: a gap is never accepted. + return 0, fmt.Errorf("%w: segment %d is a gap", ErrInvalidField, i) + case s.Ref < 0 || s.Ref >= poolLen: + return 0, fmt.Errorf("%w: segment %d references pool entry %d of %d", ErrOutOfRange, i, s.Ref, poolLen) + case s.Size > MaxSegmentSize: + return 0, fmt.Errorf("%w: segment %d is %d bytes", ErrImplausible, i, s.Size) + case s.Size < 1: + return 0, fmt.Errorf("%w: segment %d size %d", ErrOutOfRange, i, s.Size) + case s.Start < 0 || s.End < s.Start || s.End > s.Size-1: + return 0, fmt.Errorf("%w: segment %d range [%d,%d] of %d", ErrOutOfRange, i, s.Start, s.End, s.Size) + } + if s.Ref >= 0 && v.used != nil { + v.used[s.Ref] = true + } + covered += s.End - s.Start + 1 + } + return covered, nil +} + +func (v *validator) checkNested(f *File) error { + if len(f.Extents) == 0 { + return fmt.Errorf("%w: outer sources without extents", ErrInvalidField) + } + limits := make([]int64, len(f.Outer)) + for i := range f.Outer { + lim, err := v.checkOuter(&f.Outer[i]) + if err != nil { + return fmt.Errorf("outer source %d: %w", i, err) + } + limits[i] = lim + } + return checkExtents(f, limits) +} + +// checkOuter validates one outer volume and returns the number of bytes an +// extent may address in it. +func (v *validator) checkOuter(o *OuterSource) (int64, error) { + covered, err := v.checkTable(o.Segments) + if err != nil { + return 0, err + } + if len(o.Segments) == 0 { + return 0, fmt.Errorf("%w: outer source without segments", ErrInvalidField) + } + if err := checkKeyIV(o.AESKey, o.AESIV); err != nil { + return 0, err + } + if len(o.AESKey) == 0 { + if o.VolumeSize != 0 { + return 0, fmt.Errorf("%w: volume size on an unencrypted outer source", ErrBadEncryption) + } + return covered, nil + } + if o.VolumeSize < 1 || o.VolumeSize > MaxFileSize { + return 0, fmt.Errorf("%w: volume size %d", ErrBadEncryption, o.VolumeSize) + } + if covered < padded(o.VolumeSize) || covered > o.VolumeSize+aesBlockSize-1 { + return 0, fmt.Errorf("%w: encrypted volume of %d bytes covered by %d", ErrBadEncryption, o.VolumeSize, covered) + } + return o.VolumeSize, nil +} + +func checkExtents(f *File, limits []int64) error { + used := make([]bool, len(f.Outer)) + var sum int64 + for i, e := range f.Extents { + switch { + case e.Outer < 0 || e.Outer >= len(f.Outer): + return fmt.Errorf("%w: extent %d outer index %d", ErrOutOfRange, i, e.Outer) + case e.InnerLength < 1 || e.InnerLength > MaxFileSize, + e.InnerOffset < 0 || e.InnerOffset > MaxFileSize: + return fmt.Errorf("%w: extent %d [%d,+%d)", ErrOutOfRange, i, e.InnerOffset, e.InnerLength) + case e.InnerOffset+e.InnerLength > limits[e.Outer]: + return fmt.Errorf("%w: extent %d ends at %d, outer source holds %d", ErrOutOfRange, i, e.InnerOffset+e.InnerLength, limits[e.Outer]) + case sum > MaxFileSize-e.InnerLength: + return fmt.Errorf("%w: extents add up past the file cap", ErrSizeMismatch) + } + used[e.Outer] = true + sum += e.InnerLength + } + for i, u := range used { + if !u { + return fmt.Errorf("%w: outer source %d is not used by any extent", ErrInvalidField, i) + } + } + if err := checkOverlap(f.Extents); err != nil { + return err + } + if sum != f.Size { + return fmt.Errorf("%w: size %d, extents add up to %d", ErrSizeMismatch, f.Size, sum) + } + return nil +} + +// checkOverlap rejects two extents of one outer source that share bytes. +func checkOverlap(extents []Extent) error { + order := make([]int, len(extents)) + for i := range order { + order[i] = i + } + sort.Slice(order, func(a, b int) bool { + x, y := extents[order[a]], extents[order[b]] + if x.Outer != y.Outer { + return x.Outer < y.Outer + } + return x.InnerOffset < y.InnerOffset + }) + for k := 1; k < len(order); k++ { + p, c := extents[order[k-1]], extents[order[k]] + if p.Outer == c.Outer && p.InnerOffset+p.InnerLength > c.InnerOffset { + return fmt.Errorf("%w: extents %d and %d", ErrOverlap, order[k-1], order[k]) + } + } + return nil +} + +func checkClips(f *File) error { + var sum int64 + for i, c := range f.Clips { + switch { + case c.ByteLen < 1 || c.ByteLen > f.Size || c.ByteLen%ClipPacketSize != 0: + return fmt.Errorf("%w: clip %d length %d", ErrBadClip, i, c.ByteLen) + case c.Delta90k < -MaxClipDelta || c.Delta90k > MaxClipDelta: + return fmt.Errorf("%w: clip %d delta", ErrBadClip, i) + } + sum += c.ByteLen // each is <= Size <= 2^50 and there are <= 100k + } + if sum != f.Size { + return fmt.Errorf("%w: clips add up to %d of %d", ErrBadClip, sum, f.Size) + } + return nil +} + +func (v *validator) checkPar2(f *File) error { + names := make(map[string]struct{}, len(f.Par2)) + for i := range f.Par2 { + p := &f.Par2[i] + if err := checkFilename(p.Name); err != nil { + return fmt.Errorf("par2 %d: %w", i, err) + } + key := strings.ToLower(p.Name) + if _, dup := names[key]; dup { + return fmt.Errorf("%w: par2 %q", ErrDuplicateName, p.Name) + } + names[key] = struct{}{} + if p.Size < 1 || p.Size > MaxFileSize || len(p.Segments) == 0 { + return fmt.Errorf("%w: par2 %d size or segments", ErrInvalidField, i) + } + covered, err := v.checkTable(p.Segments) + if err != nil { + return fmt.Errorf("par2 %d: %w", i, err) + } + if p.Size > int64(len(p.Segments))*MaxSegmentSize { + return fmt.Errorf("%w: par2 %d", ErrImplausible, i) + } + if covered != p.Size { + return fmt.Errorf("%w: par2 %d size %d, segments cover %d", ErrSizeMismatch, i, p.Size, covered) + } + } + return nil +} + +// entryOf returns the NZB file entry that holds pool position pos. +func (v *validator) entryOf(pos int64) int { + i := sort.Search(len(v.entStart), func(k int) bool { return int64(v.entStart[k]+v.entLen[k]) > pos }) + for i < len(v.entStart) && v.entLen[i] == 0 { + i++ + } + return i +} + +// wholeEntry returns the NZB file entry a segment table reads, requiring the +// table to be exactly that entry's segments in NZB order. +func (v *validator) wholeEntry(segs []Segment) (int, error) { + if len(segs) == 0 { + return 0, fmt.Errorf("%w: empty table", ErrCoverage) + } + e := v.entryOf(segs[0].Ref) + if e >= len(v.entStart) || segs[0].Ref != int64(v.entStart[e]) || len(segs) != v.entLen[e] { + return 0, fmt.Errorf("%w: table does not start at, or does not span, one NZB file entry", ErrCoverage) + } + for i, s := range segs { + if s.Ref != int64(v.entStart[e]+i) { + return 0, fmt.Errorf("%w: segment %d is out of order or from another entry", ErrCoverage, i) + } + } + return e, nil +} + +// checkCoverage (nzb scheme) requires every segment table to be exactly one +// NZB file entry. An entry read inline belongs to one file and is read once; +// outer volumes and PAR2 files may be shared by several files, but only once +// within a file and never also inline. +func (v *validator) checkCoverage(f *File) error { + if v.b.Scheme != SchemeNZB { + return nil + } + own := map[int]struct{}{} + use := func(segs []Segment, inline bool) error { + e, err := v.wholeEntry(segs) + if err != nil { + return err + } + if _, dup := own[e]; dup { + return fmt.Errorf("%w: NZB entry %d is read twice by one file", ErrCoverage, e) + } + own[e] = struct{}{} + _, wasInline := v.inline[e] + if wasInline || (inline && v.shared(e)) { + return fmt.Errorf("%w: NZB entry %d is used inline and elsewhere", ErrCoverage, e) + } + if inline { + v.inline[e] = struct{}{} + } else { + v.markShared(e) + } + return nil + } + if len(f.Segments) > 0 { + if err := use(f.Segments, true); err != nil { + return err + } + } + for i := range f.Outer { + if err := use(f.Outer[i].Segments, false); err != nil { + return fmt.Errorf("outer source %d: %w", i, err) + } + } + for i := range f.Par2 { + if err := use(f.Par2[i].Segments, false); err != nil { + return fmt.Errorf("par2 %d: %w", i, err) + } + } + return nil +} + +func (v *validator) shared(e int) bool { _, ok := v.sharedE[e]; return ok } + +func (v *validator) markShared(e int) { v.sharedE[e] = struct{}{} } diff --git a/internal/peernet/bundle/validate_test.go b/internal/peernet/bundle/validate_test.go new file mode 100644 index 000000000..bfd2178c5 --- /dev/null +++ b/internal/peernet/bundle/validate_test.go @@ -0,0 +1,241 @@ +package bundle + +import ( + "bytes" + "errors" + "math" + "strings" + "testing" + + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +func TestValidateAcceptsGoodBundles(t *testing.T) { + for name, b := range map[string]*Bundle{"files": goodBundle(t), "nzb": goodNZBBundle(t)} { + if err := Validate(b); err != nil { + t.Errorf("%s: %v", name, err) + } + } + if err := Validate(nil); !errors.Is(err, ErrInvalidField) { + t.Errorf("nil bundle: %v", err) + } +} + +func TestValidateRejects(t *testing.T) { + tests := []struct { + name string + nzb bool + mutate func(b *Bundle) + want error + }{ + // ---- segment ranges and sums + {"file size one too big", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Size++ }, ErrSizeMismatch}, + {"end offset one past the segment", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Segments[0].End = 1000 }, ErrOutOfRange}, + {"start after end", false, func(b *Bundle) { s := &fileNamed(t, b, "movie.mkv").Segments[0]; s.Start, s.End = 500, 100 }, ErrOutOfRange}, + {"negative start", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Segments[0].Start = -1 }, ErrOutOfRange}, + {"ref past the pool", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Segments[0].Ref = int64(len(b.Pool)) }, ErrOutOfRange}, + {"ref below gap", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Segments[0].Ref = -2 }, ErrOutOfRange}, + {"segment size zero", false, func(b *Bundle) { s := &fileNamed(t, b, "movie.mkv").Segments[0]; s.Size, s.Start, s.End = 0, 0, -1 }, ErrOutOfRange}, + {"segment larger than any article", false, func(b *Bundle) { + s := &fileNamed(t, b, "movie.mkv").Segments[0] + s.Size, s.End = MaxSegmentSize+1, MaxSegmentSize + }, ErrImplausible}, + {"segment size near MaxInt64", false, func(b *Bundle) { + s := &fileNamed(t, b, "movie.mkv").Segments[0] + s.Size, s.End = math.MaxInt64, math.MaxInt64-1 + }, ErrImplausible}, + {"one byte gap inside the extents", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Segments[1].Start = 1 }, ErrSizeMismatch}, + {"declared size far beyond the segments", false, func(b *Bundle) { + f := fileNamed(t, b, "movie.mkv") + b.TotalSize += (1 << 40) - f.Size + f.Size = 1 << 40 + }, ErrImplausible}, + {"file size zero", false, func(b *Bundle) { f := fileNamed(t, b, "movie.mkv"); b.TotalSize -= f.Size; f.Size = 0 }, ErrInvalidField}, + {"file size above cap", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Size = MaxFileSize + 1 }, ErrInvalidField}, + {"file without data", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Segments = nil }, ErrInvalidField}, + {"total size mismatch", false, func(b *Bundle) { b.TotalSize++ }, ErrSizeMismatch}, + {"release date negative", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").ReleaseDate = -1 }, ErrInvalidField}, + {"release date too far", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").ReleaseDate = maxUnixSeconds + 1 }, ErrInvalidField}, + + // ---- encryption + {"aes key 15 bytes", false, func(b *Bundle) { f := fileNamed(t, b, "secret.bin"); f.AESKey = f.AESKey[:15] }, ErrBadEncryption}, + {"aes key 33 bytes", false, func(b *Bundle) { f := fileNamed(t, b, "secret.bin"); f.AESKey = append(f.AESKey, 1) }, ErrBadEncryption}, + {"aes iv 15 bytes", false, func(b *Bundle) { f := fileNamed(t, b, "secret.bin"); f.AESIV = f.AESIV[:15] }, ErrBadEncryption}, + {"aes key without iv", false, func(b *Bundle) { fileNamed(t, b, "secret.bin").AESIV = nil }, ErrBadEncryption}, + {"aes iv without key", false, func(b *Bundle) { f := fileNamed(t, b, "movie.mkv"); f.AESIV = bytes.Repeat([]byte{1}, 16) }, ErrBadEncryption}, + {"encrypted usable range shorter than padded size", false, func(b *Bundle) { fileNamed(t, b, "secret.bin").Segments[0].End-- }, ErrBadEncryption}, + {"encrypted usable range longer than padded size", false, func(b *Bundle) { fileNamed(t, b, "secret.bin").Segments[0].End += 4 }, ErrBadEncryption}, + {"aes key on a nested file", false, func(b *Bundle) { + f := fileNamed(t, b, "nested.mkv") + f.AESKey, f.AESIV = key32(), iv16() + }, ErrBadEncryption}, + {"outer key length", false, func(b *Bundle) { f := fileNamed(t, b, "nested.mkv"); f.Outer[0].AESKey = f.Outer[0].AESKey[:20] }, ErrBadEncryption}, + {"outer iv missing", false, func(b *Bundle) { fileNamed(t, b, "nested.mkv").Outer[0].AESIV = nil }, ErrBadEncryption}, + {"outer volume size zero", false, func(b *Bundle) { fileNamed(t, b, "nested.mkv").Outer[0].VolumeSize = 0 }, ErrBadEncryption}, + {"outer volume size on unencrypted outer", false, func(b *Bundle) { fileNamed(t, b, "disc.m2ts").Outer[0].VolumeSize = 5 }, ErrBadEncryption}, + {"outer coverage short of the padded volume", false, func(b *Bundle) { fileNamed(t, b, "nested.mkv").Outer[0].VolumeSize = 1100 }, ErrBadEncryption}, + + // ---- nested sources + {"extent outer index out of range", false, func(b *Bundle) { fileNamed(t, b, "nested.mkv").Extents[0].Outer = 1 }, ErrOutOfRange}, + {"extent outer index negative", false, func(b *Bundle) { fileNamed(t, b, "nested.mkv").Extents[0].Outer = -1 }, ErrOutOfRange}, + {"extent beyond the encrypted volume", false, func(b *Bundle) { fileNamed(t, b, "nested.mkv").Extents[1].InnerOffset = 801 }, ErrOutOfRange}, + {"extent beyond the plain outer coverage", false, func(b *Bundle) { fileNamed(t, b, "disc.m2ts").Extents[1].InnerOffset = 1100 }, ErrOutOfRange}, + {"extent length zero", false, func(b *Bundle) { fileNamed(t, b, "nested.mkv").Extents[0].InnerLength = 0 }, ErrOutOfRange}, + {"extent offset negative", false, func(b *Bundle) { fileNamed(t, b, "nested.mkv").Extents[0].InnerOffset = -1 }, ErrOutOfRange}, + {"extents do not sum to the file size", false, func(b *Bundle) { fileNamed(t, b, "nested.mkv").Extents[0].InnerLength-- }, ErrSizeMismatch}, + {"extents overlap", false, func(b *Bundle) { + f := fileNamed(t, b, "nested.mkv") + f.Extents[1].InnerOffset = 300 + }, ErrOverlap}, + {"sum of extents overflows int64", false, func(b *Bundle) { + f := fileNamed(t, b, "nested.mkv") + f.Extents[0].InnerLength = math.MaxInt64 - 5 + f.Extents[1].InnerLength = 10 + }, ErrOutOfRange}, + {"extent offset plus length overflows", false, func(b *Bundle) { + f := fileNamed(t, b, "nested.mkv") + f.Extents[0].InnerOffset = math.MaxInt64 - 5 + }, ErrOutOfRange}, + {"unreferenced outer source", false, func(b *Bundle) { + f := fileNamed(t, b, "nested.mkv") + f.Outer = append(f.Outer, f.Outer[0]) + }, ErrInvalidField}, + {"nested file also lists segments", false, func(b *Bundle) { + fileNamed(t, b, "nested.mkv").Segments = []Segment{{Ref: 0, Size: 10, Start: 0, End: 9}} + }, ErrInvalidField}, + {"extents without outer sources", false, func(b *Bundle) { fileNamed(t, b, "nested.mkv").Outer = nil }, ErrOutOfRange}, + {"outer sources without extents", false, func(b *Bundle) { fileNamed(t, b, "nested.mkv").Extents = nil }, ErrInvalidField}, + {"too many outer sources", false, func(b *Bundle) { + f := fileNamed(t, b, "nested.mkv") + for i := 0; i <= MaxOuterSources; i++ { + f.Outer = append(f.Outer, OuterSource{}) + } + }, ErrLimit}, + {"too many extents", false, func(b *Bundle) { + f := fileNamed(t, b, "disc.m2ts") + f.Extents = make([]Extent, MaxExtents+1) + }, ErrLimit}, + + // ---- clips + {"clips not summing to the file size", false, func(b *Bundle) { fileNamed(t, b, "disc.m2ts").Clips[1].ByteLen += ClipPacketSize }, ErrBadClip}, + {"clip not a packet multiple", false, func(b *Bundle) { + c := fileNamed(t, b, "disc.m2ts").Clips + c[0].ByteLen++ + c[1].ByteLen-- + }, ErrBadClip}, + {"clip length zero", false, func(b *Bundle) { + c := fileNamed(t, b, "disc.m2ts").Clips + c[0].ByteLen = 0 + c[1].ByteLen += 192 + }, ErrBadClip}, + {"clip delta out of range", false, func(b *Bundle) { fileNamed(t, b, "disc.m2ts").Clips[0].Delta90k = MaxClipDelta + 1 }, ErrBadClip}, + {"clip delta min int64", false, func(b *Bundle) { fileNamed(t, b, "disc.m2ts").Clips[0].Delta90k = math.MinInt64 }, ErrBadClip}, + {"too many clips", false, func(b *Bundle) { fileNamed(t, b, "disc.m2ts").Clips = make([]Clip, MaxClips+1) }, ErrLimit}, + + // ---- PAR2 + {"par2 gap", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Par2[0].Segments[0].Ref = GapRef }, ErrInvalidField}, + {"par2 size mismatch", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Par2[0].Size++ }, ErrSizeMismatch}, + {"par2 bad name", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Par2[0].Name = "../x.par2" }, ErrBadFilename}, + {"par2 duplicate name", false, func(b *Bundle) { + f := fileNamed(t, b, "movie.mkv") + f.Par2 = append(f.Par2, f.Par2[0]) + }, ErrDuplicateName}, + {"too many par2 files", false, func(b *Bundle) { + f := fileNamed(t, b, "movie.mkv") + f.Par2 = make([]Par2File, MaxPar2Files+1) + }, ErrLimit}, + + // ---- names and text + {"path traversal name", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Name = "../x" }, ErrBadFilename}, + {"slash in name", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Name = "a/b" }, ErrBadFilename}, + {"backslash in name", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Name = `a\b` }, ErrBadFilename}, + {"NUL in name", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Name = "a\x00b" }, ErrBadFilename}, + {"dot-dot name", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Name = ".." }, ErrBadFilename}, + {"windows reserved name", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Name = "CON.mkv" }, ErrBadFilename}, + {"bidi in name", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Name = "a\u202Eb" }, ErrBadFilename}, + {"long name", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Name = strings.Repeat("a", 256) }, ErrBadFilename}, + {"duplicate names", false, func(b *Bundle) { fileNamed(t, b, "secret.bin").Name = "movie.mkv" }, ErrDuplicateName}, + {"duplicate names ignoring case", false, func(b *Bundle) { fileNamed(t, b, "secret.bin").Name = "MOVIE.MKV" }, ErrDuplicateName}, + {"bad title", false, func(b *Bundle) { b.Title = "a\u202Eb" }, ErrInvalidField}, + {"empty title", false, func(b *Bundle) { b.Title = "" }, ErrInvalidField}, + {"posted zero", false, func(b *Bundle) { b.PostedUnix = 0 }, ErrInvalidField}, + {"posted too far", false, func(b *Bundle) { b.PostedUnix = maxUnixSeconds + 1 }, ErrInvalidField}, + {"password control character", false, func(b *Bundle) { b.NZBPassword = "a\x00b" }, ErrInvalidField}, + {"password too long", false, func(b *Bundle) { b.NZBPassword = strings.Repeat("p", MaxPasswordLen+1) }, ErrInvalidField}, + + // ---- ids, files, pool + {"zero release id", false, func(b *Bundle) { b.ReleaseID = contentid.ID{} }, ErrInvalidField}, + {"zero file id", false, func(b *Bundle) { b.Files[0].FileID = contentid.ID{} }, ErrInvalidField}, + {"unknown scheme", false, func(b *Bundle) { b.Scheme = 9 }, ErrUnknownScheme}, + {"no files", false, func(b *Bundle) { b.Files = nil; b.TotalSize = 0 }, ErrInvalidField}, + {"files unsorted", false, func(b *Bundle) { b.Files[0], b.Files[1] = b.Files[1], b.Files[0] }, ErrInvalidField}, + {"duplicate file id", false, func(b *Bundle) { b.Files[1].FileID = b.Files[0].FileID }, ErrDuplicateFile}, + {"pool entry with a space", false, func(b *Bundle) { b.Pool[0] = "a b@x.test" }, ErrBadMessageID}, + {"pool entry with CRLF", false, func(b *Bundle) { b.Pool[0] = "a\r\nQUIT@x" }, ErrBadMessageID}, + {"pool entry that looks like a placeholder", false, func(b *Bundle) { b.Pool[0] = "altmount-gap-3-x" }, ErrBadMessageID}, + {"pool entry too long", false, func(b *Bundle) { b.Pool[0] = strings.Repeat("a", MaxMessageIDLen+1) }, ErrBadMessageID}, + {"pool entry in angle brackets", false, func(b *Bundle) { b.Pool[0] = "<" + b.Pool[0] + ">" }, ErrBadMessageID}, + {"files scheme duplicate pool entry", false, func(b *Bundle) { b.Pool[1] = b.Pool[0] }, ErrBadPool}, + {"files scheme unreferenced pool entry", false, func(b *Bundle) { b.Pool = append(b.Pool, "extra@x.test") }, ErrBadPool}, + {"files scheme with a release list", false, func(b *Bundle) { b.ReleaseFiles = []int{1} }, ErrInvalidField}, + {"nzb scheme counts do not match the pool", true, func(b *Bundle) { b.ReleaseFiles[0]++ }, ErrBadPool}, + {"nzb scheme without a release list", true, func(b *Bundle) { b.ReleaseFiles = nil }, ErrBadPool}, + {"nzb scheme negative count", true, func(b *Bundle) { b.ReleaseFiles[0] = -1; b.ReleaseFiles[1]++ }, ErrBadPool}, + {"nzb scheme too many release files", true, func(b *Bundle) { b.ReleaseFiles = make([]int, MaxReleaseFiles+1) }, ErrLimit}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + b := goodBundle(t) + if tc.nzb { + b = goodNZBBundle(t) + } + tc.mutate(b) + err := Validate(b) + if !errors.Is(err, tc.want) { + t.Fatalf("Validate = %v, want %v", err, tc.want) + } + }) + } +} + +func TestValidateSegmentCountCaps(t *testing.T) { + const n = MaxSegments + 1 + pool := make([]string, n) + segs := make([]Segment, n) + for i := range pool { + pool[i] = "id" + strings.Repeat("x", 5) + string(rune('a'+i%26)) + string(rune('a'+(i/26)%26)) + string(rune('a'+(i/676)%26)) + string(rune('a'+(i/17576)%26)) + string(rune('a'+(i/456976)%26)) + segs[i] = Segment{Ref: int64(i), Size: 1, Start: 0, End: 0} + } + b := &Bundle{Scheme: SchemeFiles, ReleaseID: contentid.ID{1}, Title: "t", PostedUnix: 1, TotalSize: n, Pool: pool, + Files: []File{{Name: "a", Size: n, FileID: contentid.ID{2}, Segments: segs}}} + if err := Validate(b); !errors.Is(err, ErrLimit) { + t.Fatalf("500,001 segments: %v", err) + } + // Exactly at the cap the count check passes (other checks may still fail + // on this synthetic data, but never with ErrLimit). + b.Pool, b.Files[0].Segments, b.Files[0].Size, b.TotalSize = pool[:MaxSegments], segs[:MaxSegments], MaxSegments, MaxSegments + if err := Validate(b); errors.Is(err, ErrLimit) { + t.Fatalf("500,000 segments rejected as too many: %v", err) + } +} + +func TestValidateFileCountCap(t *testing.T) { + b := goodBundle(t) + for len(b.Files) <= MaxFiles { + b.Files = append(b.Files, b.Files[0]) + } + if err := Validate(b); !errors.Is(err, ErrLimit) { + t.Fatalf("10,001 files: %v", err) + } +} + +func TestValidateDoesNotMutate(t *testing.T) { + b := goodBundle(t) + c := clone(b) + if err := Validate(b); err != nil { + t.Fatal(err) + } + if !bytes.Equal(encodeUnchecked(b), encodeUnchecked(c)) { + t.Fatal("Validate changed the bundle") + } +} diff --git a/internal/peernet/bundle/verify.go b/internal/peernet/bundle/verify.go new file mode 100644 index 000000000..8865582ab --- /dev/null +++ b/internal/peernet/bundle/verify.go @@ -0,0 +1,104 @@ +package bundle + +import ( + "fmt" + + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +// Options lets the caller replace the files-scheme Release ID function, so +// this package does not depend on the one in contentid until it exists. +type Options struct { + // FilesReleaseID computes the Release ID of the files scheme from the File + // IDs, which are passed sorted ascending bytewise. Nil selects + // FilesReleaseID. + FilesReleaseID func(sortedFileIDs []contentid.ID) contentid.ID +} + +// FilesReleaseID is the files-scheme Release ID, as defined by +// contentid.ReleaseIDFromFiles: SHA-256 over the "altmount-peernet/release-files/v1" +// domain label and the File IDs sorted ascending bytewise. ids is not +// modified. An invalid input (no files, a zero File ID) yields the zero ID, +// which never equals a real Release ID. +func FilesReleaseID(ids []contentid.ID) contentid.ID { + id, err := contentid.ReleaseIDFromFiles(ids) + if err != nil { + return contentid.ID{} + } + return id +} + +// Pack encodes, validates and compresses b for the wire and returns the +// Bundle ID (the hash of the canonical uncompressed bytes). +func Pack(b *Bundle) (wire []byte, id BundleID, err error) { + raw, err := Encode(b) + if err != nil { + return nil, BundleID{}, err + } + wire, err = Compress(raw) + if err != nil { + return nil, BundleID{}, err + } + return wire, ComputeID(raw), nil +} + +// Verify checks a bundle received from a peer, in this order: wire size and +// bounded decompression; the hash equals the announced Bundle ID; strict +// decoding and structural validation; every File ID recomputed from the +// segments; the Release ID recomputed for the declared scheme equals both the +// bundle's own and the announced one. It does not touch the network. +func Verify(wire []byte, announcedBundle BundleID, announcedRelease contentid.ID) (*Bundle, error) { + return VerifyWith(wire, announcedBundle, announcedRelease, Options{}) +} + +// VerifyWith is Verify with options. +func VerifyWith(wire []byte, announcedBundle BundleID, announcedRelease contentid.ID, o Options) (*Bundle, error) { + raw, err := Decompress(wire) + if err != nil { + return nil, err + } + return VerifyCanonical(raw, announcedBundle, announcedRelease, o) +} + +// VerifyCanonical is Verify for bytes that are already decompressed. +func VerifyCanonical(canonical []byte, announcedBundle BundleID, announcedRelease contentid.ID, o Options) (*Bundle, error) { + if len(canonical) > MaxDecompressedSize { + return nil, fmt.Errorf("%w: %d bytes uncompressed", ErrTooLarge, len(canonical)) + } + if ComputeID(canonical) != announcedBundle { + return nil, ErrBundleIDMismatch + } + b, err := Decode(canonical) + if err != nil { + return nil, err + } + ids := make([]contentid.ID, len(b.Files)) + for i := range b.Files { + f := &b.Files[i] + id, err := contentid.FileID(buildMeta(b, f)) + if err != nil { + return nil, fmt.Errorf("%w: file %q: %v", ErrFileIDMismatch, f.Name, err) + } + if id != f.FileID { + return nil, fmt.Errorf("%w: file %q", ErrFileIDMismatch, f.Name) + } + ids[i] = id + } + var got contentid.ID + switch b.Scheme { + case SchemeNZB: + if got, err = releaseIDOfList(b.Pool, b.ReleaseFiles); err != nil { + return nil, fmt.Errorf("%w: %v", ErrReleaseIDMismatch, err) + } + default: + fn := o.FilesReleaseID + if fn == nil { + fn = FilesReleaseID + } + got = fn(ids) + } + if got != b.ReleaseID || got != announcedRelease { + return nil, ErrReleaseIDMismatch + } + return b, nil +} diff --git a/internal/peernet/bundle/verify_test.go b/internal/peernet/bundle/verify_test.go new file mode 100644 index 000000000..96b52f233 --- /dev/null +++ b/internal/peernet/bundle/verify_test.go @@ -0,0 +1,220 @@ +package bundle + +import ( + "bytes" + "errors" + "testing" + + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +func pack(t testing.TB, b *Bundle) ([]byte, BundleID) { + t.Helper() + wire, id, err := Pack(b) + if err != nil { + t.Fatalf("Pack: %v", err) + } + return wire, id +} + +func TestVerifyAcceptsBothSchemes(t *testing.T) { + for name, b := range map[string]*Bundle{"files": goodBundle(t), "nzb": goodNZBBundle(t)} { + wire, id := pack(t, b) + got, err := Verify(wire, id, b.ReleaseID) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if got.ReleaseID != b.ReleaseID || len(got.Files) != len(b.Files) { + t.Fatalf("%s: wrong bundle returned", name) + } + } +} + +func TestPackIDIsHashOfCanonicalBytes(t *testing.T) { + b := goodBundle(t) + wire, id := pack(t, b) + if id != ComputeID(mustEncode(t, b)) { + t.Fatal("Pack id is not the hash of the canonical bytes") + } + raw, err := Decompress(wire) + if err != nil || !bytes.Equal(raw, mustEncode(t, b)) { + t.Fatalf("wire does not decompress to the canonical bytes: %v", err) + } +} + +func TestVerifyRejects(t *testing.T) { + good := goodBundle(t) + wire, id := pack(t, good) + other := clone(good) + other.Title = "Another title" + otherWire, otherID := pack(t, other) + + badFileID := clone(good) + badFileID.Files[0].FileID[0] ^= 0xFF + // keep the files sorted so only the File ID claim is wrong + badFileID.Files = sortedByID(badFileID.Files) + badFileWire, badFileBID := pack(t, badFileID) + + // A bundle whose embedded Release ID is the announced one but whose + // content hashes to something else. + liar := clone(good) + liar.Files = liar.Files[:len(liar.Files)-1] + liar.Pool = nil + liar = rebuildPoolDropping(t, liar) + liarWire, liarBID := pack(t, liar) + + nzbStyle := nzbStyleReleaseID(t, good) + nzbB := goodNZBBundle(t) + nzbWire, nzbBID := pack(t, nzbB) + filesStyleOfNZB := testFilesRelease(fileIDs(nzbB)) + + junk, _ := Compress([]byte("not a bundle at all")) + + tests := []struct { + name string + wire []byte + bid BundleID + rid contentid.ID + want error + }{ + {"substituted bundle", otherWire, id, good.ReleaseID, ErrBundleIDMismatch}, + {"substituted bundle, valid for its own id but not the announced release", otherWire, otherID, contentid.ID{9}, ErrReleaseIDMismatch}, + {"wrong announced release id", wire, id, contentid.ID{9}, ErrReleaseIDMismatch}, + {"files bundle with an nzb-style release id", wire, id, nzbStyle, ErrReleaseIDMismatch}, + {"nzb bundle with a files-style release id", nzbWire, nzbBID, filesStyleOfNZB, ErrReleaseIDMismatch}, + {"wrong file id inside the bundle", badFileWire, badFileBID, good.ReleaseID, ErrFileIDMismatch}, + {"embedded release id differs from what the content hashes to", liarWire, liarBID, liar.ReleaseID, ErrReleaseIDMismatch}, + {"announced bundle id checked before decoding", junk, BundleID{1}, good.ReleaseID, ErrBundleIDMismatch}, + {"junk with its own correct id", junk, ComputeID([]byte("not a bundle at all")), good.ReleaseID, ErrMalformed}, + {"oversized wire", make([]byte, MaxCompressedSize+1), id, good.ReleaseID, ErrTooLarge}, + {"empty wire", nil, id, good.ReleaseID, ErrMalformed}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if _, err := Verify(tc.wire, tc.bid, tc.rid); !errors.Is(err, tc.want) { + t.Fatalf("Verify = %v, want %v", err, tc.want) + } + }) + } +} + +func sortedByID(fs []File) []File { + out := append([]File(nil), fs...) + for i := 1; i < len(out); i++ { + for j := i; j > 0 && bytes.Compare(out[j-1].FileID[:], out[j].FileID[:]) > 0; j-- { + out[j-1], out[j] = out[j], out[j-1] + } + } + return out +} + +func fileIDs(b *Bundle) []contentid.ID { + var ids []contentid.ID + for _, f := range b.Files { + ids = append(ids, f.FileID) + } + return ids +} + +// nzbStyleReleaseID is the nzb-scheme Release ID over the bundle's segments +// laid out one NZB file per bundle file. +func nzbStyleReleaseID(t testing.TB, b *Bundle) contentid.ID { + t.Helper() + var files [][]string + for _, f := range b.Files { + var ids []string + for _, s := range f.Segments { + if s.Ref != GapRef { + ids = append(ids, b.Pool[s.Ref]) + } + } + files = append(files, ids) + } + id, err := contentid.ReleaseID(nzbStore(files)) + if err != nil { + t.Fatal(err) + } + return id +} + +// rebuildPoolDropping keeps b.ReleaseID (now stale) but rebuilds a valid pool +// for the remaining files. +func rebuildPoolDropping(t testing.TB, b *Bundle) *Bundle { + t.Helper() + var specs []specFile + keep := map[string]bool{} + for _, f := range b.Files { + keep[f.Name] = true + } + for _, s := range testSpecs() { + if keep[s.name] { + specs = append(specs, s) + } + } + n := assemble(t, specs) + n.ReleaseID = b.ReleaseID + return n +} + +func TestVerifyWithCustomFilesReleaseID(t *testing.T) { + b := goodBundle(t) + custom := contentid.ID{7, 7, 7} + b.ReleaseID = custom + wire, id := pack(t, b) + called := 0 + opts := Options{FilesReleaseID: func(ids []contentid.ID) contentid.ID { + called++ + if len(ids) != len(b.Files) { + t.Errorf("got %d ids", len(ids)) + } + for i := 1; i < len(ids); i++ { + if bytes.Compare(ids[i-1][:], ids[i][:]) >= 0 { + t.Error("ids not sorted ascending") + } + } + return custom + }} + if _, err := VerifyWith(wire, id, custom, opts); err != nil || called != 1 { + t.Fatalf("err %v called %d", err, called) + } + if _, err := Verify(wire, id, custom); !errors.Is(err, ErrReleaseIDMismatch) { + t.Fatalf("default function must not accept the custom id: %v", err) + } +} + +func TestFilesReleaseID(t *testing.T) { + ids := []contentid.ID{{3}, {1}, {2}} + want := testFilesRelease(ids) + if got := FilesReleaseID(ids); got != want { + t.Fatalf("got %s want %s", got, want) + } + if ids[0] != (contentid.ID{3}) { + t.Fatal("input was reordered") + } + if FilesReleaseID([]contentid.ID{{1}}) == FilesReleaseID([]contentid.ID{{2}}) { + t.Fatal("collision") + } +} + +func TestVerifyCanonical(t *testing.T) { + b := goodNZBBundle(t) + raw := mustEncode(t, b) + if _, err := VerifyCanonical(raw, ComputeID(raw), b.ReleaseID, Options{}); err != nil { + t.Fatal(err) + } + if _, err := VerifyCanonical(raw, BundleID{}, b.ReleaseID, Options{}); !errors.Is(err, ErrBundleIDMismatch) { + t.Fatal(err) + } +} + +func TestVerifyNZBSchemeRecomputesTheFullList(t *testing.T) { + // Dropping the non-imported NZB file changes the release id, so a peer + // cannot trim the list and keep the announced identity. + b := goodNZBBundle(t) + trimmed := assembleNZB(t, testSpecs(), nzbLayout()[:5]) + trimmed.ReleaseID = b.ReleaseID + wire, id := pack(t, trimmed) + if _, err := Verify(wire, id, b.ReleaseID); !errors.Is(err, ErrReleaseIDMismatch) { + t.Fatalf("got %v", err) + } +} diff --git a/internal/peernet/bundle/wire.go b/internal/peernet/bundle/wire.go new file mode 100644 index 000000000..de61f0616 --- /dev/null +++ b/internal/peernet/bundle/wire.go @@ -0,0 +1,225 @@ +package bundle + +import ( + "crypto/sha256" + "fmt" + + "google.golang.org/protobuf/encoding/protowire" +) + +// Field numbers. The layout is canonical: fields appear in ascending number, +// repeated fields are consecutive, optional fields are omitted when empty. +const ( + // Bundle. + fVersion protowire.Number = 1 + fScheme protowire.Number = 2 + fReleaseID protowire.Number = 3 + fTitle protowire.Number = 4 + fPosted protowire.Number = 5 + fTotal protowire.Number = 6 + fPassword protowire.Number = 7 + fPool protowire.Number = 8 + fReleaseFiles protowire.Number = 9 + fFiles protowire.Number = 10 + + // File. + fName protowire.Number = 1 + fSize protowire.Number = 2 + fDate protowire.Number = 3 + fSegs protowire.Number = 4 + fKey protowire.Number = 5 + fIV protowire.Number = 6 + fOuter protowire.Number = 7 + fExtents protowire.Number = 8 + fClips protowire.Number = 9 + fPar2 protowire.Number = 10 + fFileID protowire.Number = 11 + + // OuterSource. + fOuterSegs protowire.Number = 1 + fOuterKey protowire.Number = 2 + fOuterIV protowire.Number = 3 + fOuterVol protowire.Number = 4 + + // Par2File. + fPar2Name protowire.Number = 1 + fPar2Size protowire.Number = 2 + fPar2Segs protowire.Number = 3 + + // Segment table. + fTabCount protowire.Number = 1 + fTabRefs protowire.Number = 2 + fTabSizes protowire.Number = 3 + fTabPartials protowire.Number = 4 +) + +// ComputeID returns the Bundle ID: the SHA-256 of the canonical bytes. +func ComputeID(canonical []byte) BundleID { return sha256.Sum256(canonical) } + +// ---- encoding ---- + +func appendBytes(b []byte, num protowire.Number, v []byte) []byte { + return protowire.AppendBytes(protowire.AppendTag(b, num, protowire.BytesType), v) +} + +func appendString(b []byte, num protowire.Number, v string) []byte { + return protowire.AppendString(protowire.AppendTag(b, num, protowire.BytesType), v) +} + +func appendVarint(b []byte, num protowire.Number, v uint64) []byte { + return protowire.AppendVarint(protowire.AppendTag(b, num, protowire.VarintType), v) +} + +func zigzag(v int64) uint64 { return uint64(v<<1) ^ uint64(v>>63) } +func unzigzag(v uint64) int64 { return int64(v>>1) ^ -int64(v&1) } + +func isFull(s Segment) bool { return s.Start == 0 && s.End == s.Size-1 } + +// appendTable encodes a segment table: position count, reference runs, size +// runs and the partial-use exceptions. +func appendTable(segs []Segment) []byte { + var out, refs, sizes, partials []byte + out = appendVarint(out, fTabCount, uint64(len(segs))) + + for i := 0; i < len(segs); { + j := i + 1 + for j < len(segs) && sameRun(segs[j-1], segs[j]) { + j++ + } + base := uint64(0) + if segs[i].Ref != GapRef { + base = uint64(segs[i].Ref) + 1 + } + refs = protowire.AppendVarint(refs, base) + refs = protowire.AppendVarint(refs, uint64(j-i)) + i = j + } + out = appendBytes(out, fTabRefs, refs) + + for i := 0; i < len(segs); { + j := i + 1 + for j < len(segs) && segs[j].Size == segs[i].Size { + j++ + } + sizes = protowire.AppendVarint(sizes, uint64(segs[i].Size)) + sizes = protowire.AppendVarint(sizes, uint64(j-i)) + i = j + } + out = appendBytes(out, fTabSizes, sizes) + + prev := -1 + for i, s := range segs { + if isFull(s) { + continue + } + partials = protowire.AppendVarint(partials, uint64(i-prev-1)) + partials = protowire.AppendVarint(partials, uint64(s.Start)) + partials = protowire.AppendVarint(partials, uint64(s.End)) + prev = i + } + if len(partials) > 0 { + out = appendBytes(out, fTabPartials, partials) + } + return out +} + +// sameRun reports whether b continues the reference run that a is part of. +func sameRun(a, b Segment) bool { + if a.Ref == GapRef { + return b.Ref == GapRef + } + return b.Ref == a.Ref+1 +} + +func appendFile(f *File) []byte { + var b []byte + b = appendString(b, fName, f.Name) + b = appendVarint(b, fSize, uint64(f.Size)) + if f.ReleaseDate != 0 { + b = appendVarint(b, fDate, uint64(f.ReleaseDate)) + } + if len(f.Segments) > 0 { + b = appendBytes(b, fSegs, appendTable(f.Segments)) + } + if len(f.AESKey) > 0 { + b = appendBytes(b, fKey, f.AESKey) + b = appendBytes(b, fIV, f.AESIV) + } + for i := range f.Outer { + o := &f.Outer[i] + var ob []byte + ob = appendBytes(ob, fOuterSegs, appendTable(o.Segments)) + if len(o.AESKey) > 0 { + ob = appendBytes(ob, fOuterKey, o.AESKey) + ob = appendBytes(ob, fOuterIV, o.AESIV) + ob = appendVarint(ob, fOuterVol, uint64(o.VolumeSize)) + } + b = appendBytes(b, fOuter, ob) + } + if len(f.Extents) > 0 { + var eb []byte + for _, e := range f.Extents { + eb = protowire.AppendVarint(eb, uint64(e.Outer)) + eb = protowire.AppendVarint(eb, uint64(e.InnerOffset)) + eb = protowire.AppendVarint(eb, uint64(e.InnerLength)) + } + b = appendBytes(b, fExtents, eb) + } + if len(f.Clips) > 0 { + var cb []byte + for _, c := range f.Clips { + cb = protowire.AppendVarint(cb, uint64(c.ByteLen)) + cb = protowire.AppendVarint(cb, zigzag(c.Delta90k)) + } + b = appendBytes(b, fClips, cb) + } + for i := range f.Par2 { + p := &f.Par2[i] + var pb []byte + pb = appendString(pb, fPar2Name, p.Name) + pb = appendVarint(pb, fPar2Size, uint64(p.Size)) + pb = appendBytes(pb, fPar2Segs, appendTable(p.Segments)) + b = appendBytes(b, fPar2, pb) + } + return appendBytes(b, fFileID, f.FileID[:]) +} + +// encodeUnchecked writes the canonical bytes without validating b. Callers +// validate first; the decoder uses it only for the canonical re-encode check. +func encodeUnchecked(b *Bundle) []byte { + var out []byte + out = appendVarint(out, fVersion, SchemaVersion) + out = appendVarint(out, fScheme, uint64(b.Scheme)) + out = appendBytes(out, fReleaseID, b.ReleaseID[:]) + out = appendString(out, fTitle, b.Title) + out = appendVarint(out, fPosted, uint64(b.PostedUnix)) + out = appendVarint(out, fTotal, uint64(b.TotalSize)) + if b.NZBPassword != "" { + out = appendString(out, fPassword, b.NZBPassword) + } + for _, id := range b.Pool { + out = appendString(out, fPool, id) + } + if b.Scheme == SchemeNZB { + var rb []byte + for _, n := range b.ReleaseFiles { + rb = protowire.AppendVarint(rb, uint64(n)) + } + out = appendBytes(out, fReleaseFiles, rb) + } + for i := range b.Files { + out = appendBytes(out, fFiles, appendFile(&b.Files[i])) + } + return out +} + +// Encode validates b and returns its canonical uncompressed bytes. +func Encode(b *Bundle) ([]byte, error) { + if b == nil { + return nil, fmt.Errorf("%w: nil bundle", ErrInvalidField) + } + if err := Validate(b); err != nil { + return nil, err + } + return encodeUnchecked(b), nil +} diff --git a/internal/peernet/bundle/wire_decode.go b/internal/peernet/bundle/wire_decode.go new file mode 100644 index 000000000..4560d5112 --- /dev/null +++ b/internal/peernet/bundle/wire_decode.go @@ -0,0 +1,656 @@ +package bundle + +import ( + "bytes" + "fmt" + "math" + + "google.golang.org/protobuf/encoding/protowire" +) + +// ---- decoding ---- + +type field struct { + num protowire.Number + typ protowire.Type + v uint64 + b []byte +} + +// fieldReader walks one message: ascending field numbers, and only the +// numbers accepted by repeated may recur. +type fieldReader struct { + b []byte + prev protowire.Number + repeated func(protowire.Number) bool +} + +func (r *fieldReader) next() (field, bool, error) { + if len(r.b) == 0 { + return field{}, false, nil + } + num, typ, n := protowire.ConsumeTag(r.b) + if n < 0 { + return field{}, false, fmt.Errorf("%w: bad tag", ErrMalformed) + } + r.b = r.b[n:] + if num < r.prev || (num == r.prev && (r.repeated == nil || !r.repeated(num))) { + return field{}, false, fmt.Errorf("%w: field %d out of order or duplicated", ErrMalformed, num) + } + r.prev = num + f := field{num: num, typ: typ} + switch typ { + case protowire.VarintType: + f.v, n = protowire.ConsumeVarint(r.b) + case protowire.BytesType: + f.b, n = protowire.ConsumeBytes(r.b) + default: + return field{}, false, fmt.Errorf("%w: unsupported wire type", ErrMalformed) + } + if n < 0 { + return field{}, false, fmt.Errorf("%w: truncated field %d", ErrMalformed, num) + } + r.b = r.b[n:] + return f, true, nil +} + +func bytesField(f field, name string) ([]byte, error) { + if f.typ != protowire.BytesType { + return nil, fmt.Errorf("%w: %s wire type", ErrMalformed, name) + } + return f.b, nil +} + +func varintField(f field, max uint64, name string) (uint64, error) { + if f.typ != protowire.VarintType { + return 0, fmt.Errorf("%w: %s wire type", ErrMalformed, name) + } + if f.v > max { + return 0, fmt.Errorf("%w: %s out of range", ErrInvalidField, name) + } + return f.v, nil +} + +func int64Field(f field, name string) (int64, error) { + v, err := varintField(f, math.MaxInt64, name) + return int64(v), err +} + +func fixedField(f field, dst []byte, name string) error { + b, err := bytesField(f, name) + if err != nil { + return err + } + if len(b) != len(dst) { + return fmt.Errorf("%w: %s length", ErrInvalidField, name) + } + copy(dst, b) + return nil +} + +func stringField(f field, name string) (string, error) { + b, err := bytesField(f, name) + return string(b), err +} + +// varReader reads consecutive varints from a packed field. +type varReader struct{ b []byte } + +func (r *varReader) more() bool { return len(r.b) > 0 } + +func (r *varReader) next() (uint64, error) { + v, n := protowire.ConsumeVarint(r.b) + if n < 0 { + return 0, fmt.Errorf("%w: bad packed varint", ErrMalformed) + } + r.b = r.b[n:] + return v, nil +} + +func (r *varReader) nextInt64() (int64, error) { + v, err := r.next() + if err != nil { + return 0, err + } + if v > math.MaxInt64 { + return 0, fmt.Errorf("%w: packed value out of range", ErrInvalidField) + } + return int64(v), nil +} + +// decoder carries the running budgets shared by every table of one bundle. +type decoder struct { + segBudget int + extentBudget int + clipBudget int + outerBudget int + par2Budget int + poolLen int64 +} + +// take deducts n from a bundle-wide budget, failing before anything +// proportional to n is allocated. +func take(budget *int, n int, what string, limit int) error { + if n > *budget { + return fmt.Errorf("%w: more than %d %s in the bundle", ErrLimit, limit, what) + } + *budget -= n + return nil +} + +// countVarints counts the varints of a packed field without decoding them. +func countVarints(b []byte) int { + n := 0 + for _, c := range b { + if c < 0x80 { + n++ + } + } + return n +} + +func (d *decoder) takeSegments(n uint64) error { + if n > uint64(d.segBudget) { + return fmt.Errorf("%w: more than %d segments", ErrLimit, MaxSegments) + } + d.segBudget -= int(n) + return nil +} + +// decodeTable expands a segment table. Every claim is checked against the +// budget before anything proportional to it is allocated. +func (d *decoder) decodeTable(b []byte) ([]Segment, error) { + r := fieldReader{b: b} + var count uint64 + var refs, sizes, partials []byte + seen := map[protowire.Number]bool{} + for { + f, ok, err := r.next() + if err != nil { + return nil, err + } + if !ok { + break + } + seen[f.num] = true + switch f.num { + case fTabCount: + if count, err = varintField(f, math.MaxUint64, "table count"); err != nil { + return nil, err + } + if err = d.takeSegments(count); err != nil { + return nil, err + } + case fTabRefs: + refs, err = bytesField(f, "table refs") + case fTabSizes: + sizes, err = bytesField(f, "table sizes") + case fTabPartials: + partials, err = bytesField(f, "table partials") + default: + err = fmt.Errorf("%w: unknown table field %d", ErrMalformed, f.num) + } + if err != nil { + return nil, err + } + } + if !seen[fTabCount] || count == 0 || !seen[fTabRefs] || !seen[fTabSizes] { + return nil, fmt.Errorf("%w: incomplete segment table", ErrMalformed) + } + segs := make([]Segment, count) + + pos, rr := uint64(0), varReader{b: refs} + for rr.more() { + base, err := rr.next() + if err != nil { + return nil, err + } + n, err := rr.next() + if err != nil { + return nil, err + } + if n == 0 || n > count-pos { + return nil, fmt.Errorf("%w: reference run overruns the table", ErrMalformed) + } + if base > uint64(d.poolLen) || (base != 0 && base-1+n > uint64(d.poolLen)) { + return nil, fmt.Errorf("%w: reference run past the pool", ErrOutOfRange) + } + for k := uint64(0); k < n; k++ { + if base == 0 { + segs[pos+k].Ref = GapRef + } else { + segs[pos+k].Ref = int64(base - 1 + k) + } + } + pos += n + } + if pos != count { + return nil, fmt.Errorf("%w: reference runs cover %d of %d positions", ErrMalformed, pos, count) + } + + pos, rr = 0, varReader{b: sizes} + for rr.more() { + size, err := rr.nextInt64() + if err != nil { + return nil, err + } + n, err := rr.next() + if err != nil { + return nil, err + } + if n == 0 || n > count-pos { + return nil, fmt.Errorf("%w: size run overruns the table", ErrMalformed) + } + for k := uint64(0); k < n; k++ { + segs[pos+k].Size = size + segs[pos+k].End = size - 1 + } + pos += n + } + if pos != count { + return nil, fmt.Errorf("%w: size runs cover %d of %d positions", ErrMalformed, pos, count) + } + + idx, rr := int64(-1), varReader{b: partials} + for rr.more() { + delta, err := rr.next() + if err != nil { + return nil, err + } + if delta >= count { + return nil, fmt.Errorf("%w: partial index past the table", ErrMalformed) + } + idx += int64(delta) + 1 + if uint64(idx) >= count { + return nil, fmt.Errorf("%w: partial index past the table", ErrMalformed) + } + if segs[idx].Start, err = rr.nextInt64(); err != nil { + return nil, err + } + if segs[idx].End, err = rr.nextInt64(); err != nil { + return nil, err + } + } + return segs, nil +} + +func (d *decoder) decodeOuter(b []byte) (OuterSource, error) { + var o OuterSource + r := fieldReader{b: b} + hasSegs := false + for { + f, ok, err := r.next() + if err != nil { + return o, err + } + if !ok { + break + } + switch f.num { + case fOuterSegs: + var tb []byte + if tb, err = bytesField(f, "outer segments"); err == nil { + o.Segments, err = d.decodeTable(tb) + hasSegs = true + } + case fOuterKey: + o.AESKey, err = copyBytes(f, "outer key") + case fOuterIV: + o.AESIV, err = copyBytes(f, "outer iv") + case fOuterVol: + o.VolumeSize, err = int64Field(f, "outer volume size") + default: + err = fmt.Errorf("%w: unknown outer source field %d", ErrMalformed, f.num) + } + if err != nil { + return o, err + } + } + if !hasSegs { + return o, fmt.Errorf("%w: outer source without segments", ErrMalformed) + } + return o, nil +} + +func (d *decoder) decodePar2(b []byte) (Par2File, error) { + var p Par2File + r := fieldReader{b: b} + hasSegs := false + for { + f, ok, err := r.next() + if err != nil { + return p, err + } + if !ok { + break + } + switch f.num { + case fPar2Name: + p.Name, err = stringField(f, "par2 name") + case fPar2Size: + p.Size, err = int64Field(f, "par2 size") + case fPar2Segs: + var tb []byte + if tb, err = bytesField(f, "par2 segments"); err == nil { + p.Segments, err = d.decodeTable(tb) + hasSegs = true + } + default: + err = fmt.Errorf("%w: unknown par2 field %d", ErrMalformed, f.num) + } + if err != nil { + return p, err + } + } + if !hasSegs { + return p, fmt.Errorf("%w: par2 file without segments", ErrMalformed) + } + return p, nil +} + +func copyBytes(f field, name string) ([]byte, error) { + b, err := bytesField(f, name) + if err != nil { + return nil, err + } + return append([]byte(nil), b...), nil +} + +func isRepeatedFileField(n protowire.Number) bool { return n == fOuter || n == fPar2 } + +func (d *decoder) decodeFile(b []byte) (File, error) { + var f File + r := fieldReader{b: b, repeated: isRepeatedFileField} + var haveName, haveSize, haveID bool + for { + fl, ok, err := r.next() + if err != nil { + return f, err + } + if !ok { + break + } + switch fl.num { + case fName: + f.Name, err = stringField(fl, "file name") + haveName = true + case fSize: + f.Size, err = int64Field(fl, "file size") + haveSize = true + case fDate: + f.ReleaseDate, err = int64Field(fl, "release date") + case fSegs: + var tb []byte + if tb, err = bytesField(fl, "file segments"); err == nil { + f.Segments, err = d.decodeTable(tb) + } + case fKey: + f.AESKey, err = copyBytes(fl, "file key") + case fIV: + f.AESIV, err = copyBytes(fl, "file iv") + case fOuter: + if len(f.Outer) >= MaxOuterSources { + return f, fmt.Errorf("%w: outer sources", ErrLimit) + } + if err = take(&d.outerBudget, 1, "outer sources", MaxTotalOuterSources); err != nil { + return f, err + } + var ob []byte + if ob, err = bytesField(fl, "outer source"); err == nil { + var o OuterSource + if o, err = d.decodeOuter(ob); err == nil { + f.Outer = append(f.Outer, o) + } + } + case fExtents: + f.Extents, err = d.decodeExtents(fl) + case fClips: + f.Clips, err = d.decodeClips(fl) + case fPar2: + if len(f.Par2) >= MaxPar2Files { + return f, fmt.Errorf("%w: par2 files", ErrLimit) + } + if err = take(&d.par2Budget, 1, "par2 files", MaxTotalPar2Files); err != nil { + return f, err + } + var pb []byte + if pb, err = bytesField(fl, "par2 file"); err == nil { + var p Par2File + if p, err = d.decodePar2(pb); err == nil { + f.Par2 = append(f.Par2, p) + } + } + case fFileID: + err = fixedField(fl, f.FileID[:], "file id") + haveID = true + default: + err = fmt.Errorf("%w: unknown file field %d", ErrMalformed, fl.num) + } + if err != nil { + return f, err + } + } + if !haveName || !haveSize || !haveID { + return f, fmt.Errorf("%w: incomplete file entry", ErrMalformed) + } + return f, nil +} + +func (d *decoder) decodeExtents(f field) ([]Extent, error) { + b, err := bytesField(f, "extents") + if err != nil { + return nil, err + } + var out []Extent + r := varReader{b: b} + for r.more() { + if d.extentBudget == 0 { + return nil, fmt.Errorf("%w: more than %d extents", ErrLimit, MaxExtents) + } + d.extentBudget-- + outer, err := r.next() + if err != nil { + return nil, err + } + if outer > MaxOuterSources { + return nil, fmt.Errorf("%w: extent outer index", ErrOutOfRange) + } + var e Extent + e.Outer = int(outer) + if e.InnerOffset, err = r.nextInt64(); err != nil { + return nil, err + } + if e.InnerLength, err = r.nextInt64(); err != nil { + return nil, err + } + out = append(out, e) + } + return out, nil +} + +func (d *decoder) decodeClips(f field) ([]Clip, error) { + b, err := bytesField(f, "clips") + if err != nil { + return nil, err + } + // Count the packed varints first so the allocation is bounded by the + // budgets, not by what the input claims. + nv := countVarints(b) + if nv%2 != 0 { + return nil, fmt.Errorf("%w: odd clip varint count", ErrMalformed) + } + if nv/2 > MaxClips { + return nil, fmt.Errorf("%w: clips", ErrLimit) + } + if err := take(&d.clipBudget, nv/2, "clips", MaxTotalClips); err != nil { + return nil, err + } + out := make([]Clip, 0, nv/2) + r := varReader{b: b} + for r.more() { + var c Clip + if c.ByteLen, err = r.nextInt64(); err != nil { + return nil, err + } + z, err := r.next() + if err != nil { + return nil, err + } + c.Delta90k = unzigzag(z) + out = append(out, c) + } + return out, nil +} + +func decodeReleaseFiles(f field) ([]int, error) { + b, err := bytesField(f, "release files") + if err != nil { + return nil, err + } + var out []int + r := varReader{b: b} + for r.more() { + if len(out) >= MaxReleaseFiles { + return nil, fmt.Errorf("%w: release files", ErrLimit) + } + v, err := r.next() + if err != nil { + return nil, err + } + if v > MaxSegments { + return nil, fmt.Errorf("%w: release file segment count", ErrLimit) + } + out = append(out, int(v)) + } + return out, nil +} + +const ( + haveVersion = 1 << iota + haveScheme + haveRelease + haveTitle + havePosted + haveTotal + haveFiles + haveAll = haveVersion | haveScheme | haveRelease | haveTitle | havePosted | haveTotal | haveFiles +) + +func isRepeatedBundleField(n protowire.Number) bool { return n == fPool || n == fFiles } + +// decodeRaw parses the wire layout strictly (order, duplicates, resource +// budgets) without the structural checks of Validate. +func decodeRaw(raw []byte) (*Bundle, error) { + if len(raw) > MaxDecompressedSize { + return nil, fmt.Errorf("%w: %d bytes", ErrTooLarge, len(raw)) + } + if len(raw) == 0 { + return nil, fmt.Errorf("%w: empty input", ErrMalformed) + } + b := &Bundle{} + d := &decoder{segBudget: MaxSegments, extentBudget: MaxExtents, clipBudget: MaxTotalClips, + outerBudget: MaxTotalOuterSources, par2Budget: MaxTotalPar2Files} + var val *validator + r := fieldReader{b: raw, repeated: isRepeatedBundleField} + have := 0 + for { + f, ok, err := r.next() + if err != nil { + return nil, err + } + if !ok { + break + } + switch f.num { + case fVersion: + var v uint64 + if v, err = varintField(f, math.MaxUint64, "version"); err == nil && v != SchemaVersion { + err = fmt.Errorf("%w: %d", ErrUnsupportedVersion, v) + } + have |= haveVersion + case fScheme: + var v uint64 + if v, err = varintField(f, math.MaxUint64, "scheme"); err == nil { + if b.Scheme = Scheme(v); v != uint64(SchemeNZB) && v != uint64(SchemeFiles) { + err = fmt.Errorf("%w: %d", ErrUnknownScheme, v) + } + } + have |= haveScheme + case fReleaseID: + err = fixedField(f, b.ReleaseID[:], "release id") + have |= haveRelease + case fTitle: + b.Title, err = stringField(f, "title") + have |= haveTitle + case fPosted: + b.PostedUnix, err = int64Field(f, "posted") + have |= havePosted + case fTotal: + b.TotalSize, err = int64Field(f, "total size") + have |= haveTotal + case fPassword: + b.NZBPassword, err = stringField(f, "password") + case fPool: + if len(b.Pool) >= MaxSegments { + return nil, fmt.Errorf("%w: pool", ErrLimit) + } + var s string + if s, err = stringField(f, "pool entry"); err == nil { + b.Pool = append(b.Pool, s) + } + case fReleaseFiles: + b.ReleaseFiles, err = decodeReleaseFiles(f) + case fFiles: + if len(b.Files) >= MaxFiles { + return nil, fmt.Errorf("%w: files", ErrLimit) + } + d.poolLen = int64(len(b.Pool)) + var fb []byte + if fb, err = bytesField(f, "file"); err == nil { + var file File + if file, err = d.decodeFile(fb); err == nil { + // Validate each file as soon as it is decoded so a + // hostile bundle fails before the next one is expanded. + if val == nil { + val = &validator{b: b, names: map[string]struct{}{}} + err = val.checkPool() + } + if err == nil { + err = val.checkFile(&file) + } + if err == nil { + b.Files = append(b.Files, file) + } + } + } + have |= haveFiles + default: + err = fmt.Errorf("%w: unknown field %d", ErrMalformed, f.num) + } + if err != nil { + return nil, err + } + } + if have != haveAll { + return nil, fmt.Errorf("%w: missing required field", ErrMalformed) + } + return b, nil +} + +// Decode strictly decodes canonical uncompressed bundle bytes and validates +// the result. Anything that is not exactly what Encode would produce for the +// decoded bundle is rejected, so a bundle has one byte representation and one +// Bundle ID. +func Decode(raw []byte) (*Bundle, error) { + b, err := decodeRaw(raw) + if err != nil { + return nil, err + } + if err := Validate(b); err != nil { + return nil, err + } + if !bytes.Equal(encodeUnchecked(b), raw) { + return nil, fmt.Errorf("%w: not the canonical encoding", ErrMalformed) + } + return b, nil +} diff --git a/internal/peernet/bundle/wire_test.go b/internal/peernet/bundle/wire_test.go new file mode 100644 index 000000000..98c63cbbe --- /dev/null +++ b/internal/peernet/bundle/wire_test.go @@ -0,0 +1,224 @@ +package bundle + +import ( + "bytes" + "errors" + "github.com/stretchr/testify/assert" + "testing" + + "google.golang.org/protobuf/encoding/protowire" +) + +// rawField is one top-level wire field, for crafting hostile encodings. +type rawField struct { + num protowire.Number + typ protowire.Type + v uint64 + b []byte +} + +func splitTop(t testing.TB, b []byte) []rawField { + t.Helper() + var out []rawField + for len(b) > 0 { + num, typ, n := protowire.ConsumeTag(b) + if n < 0 { + t.Fatal("bad tag") + } + b = b[n:] + f := rawField{num: num, typ: typ} + switch typ { + case protowire.VarintType: + f.v, n = protowire.ConsumeVarint(b) + case protowire.BytesType: + f.b, n = protowire.ConsumeBytes(b) + default: + t.Fatal("unexpected wire type") + } + if n < 0 { + t.Fatal("bad field") + } + b = b[n:] + out = append(out, f) + } + return out +} + +func joinTop(fs []rawField) []byte { + var b []byte + for _, f := range fs { + b = protowire.AppendTag(b, f.num, f.typ) + if f.typ == protowire.VarintType { + b = protowire.AppendVarint(b, f.v) + } else { + b = protowire.AppendBytes(b, f.b) + } + } + return b +} + +func mustEncode(t testing.TB, b *Bundle) []byte { + t.Helper() + raw, err := Encode(b) + if err != nil { + t.Fatalf("Encode: %v", err) + } + return raw +} + +func TestEncodeDecodeRoundTrip(t *testing.T) { + b := goodBundle(t) + raw := mustEncode(t, b) + got, err := Decode(raw) + if err != nil { + t.Fatalf("Decode: %v", err) + } + assert.Equal(t, b, got, "round trip changed the bundle") + again, err := Encode(got) + if err != nil || !bytes.Equal(again, raw) { + t.Fatalf("re-encode differs (err %v)", err) + } +} + +func TestEncodeIsDeterministicAndCompact(t *testing.T) { + b := goodBundle(t) + a, c := mustEncode(t, b), mustEncode(t, b) + if !bytes.Equal(a, c) { + t.Fatal("two encodings of one bundle differ") + } + if len(a) == 0 || len(a) > 1500 { + t.Fatalf("unexpected size %d", len(a)) + } +} + +func TestEncodeRunLengthKeepsUniformFilesTiny(t *testing.T) { + // 10,000 equal-size full segments must not cost per-segment bytes beyond + // the message-ids themselves. + b := syntheticBundle(t, 10_000, 716_800) + raw := mustEncode(t, b) + idBytes := 0 + for _, id := range b.Pool { + idBytes += len(id) + 2 + } + if overhead := len(raw) - idBytes; overhead > 400 { + t.Fatalf("layout overhead %d bytes for 10k uniform segments", overhead) + } +} + +func TestEncodeRejectsInvalidBundle(t *testing.T) { + b := goodBundle(t) + b.Files[0].Size++ + if _, err := Encode(b); !errors.Is(err, ErrSizeMismatch) && !errors.Is(err, ErrFileIDMismatch) && !errors.Is(err, ErrInvalidField) { + t.Fatalf("Encode of an invalid bundle: %v", err) + } + if _, err := Encode(nil); err == nil { + t.Fatal("Encode(nil) succeeded") + } +} + +func TestDecodeRejectsNonCanonical(t *testing.T) { + canon := mustEncode(t, goodBundle(t)) + fields := splitTop(t, canon) + mut := func(f func(fs []rawField) []rawField) []byte { + cp := append([]rawField(nil), fields...) + return joinTop(f(cp)) + } + tests := []struct { + name string + raw []byte + want error + }{ + {"empty", nil, ErrMalformed}, + {"trailing zero byte", append(append([]byte(nil), canon...), 0), ErrMalformed}, + {"truncated", canon[:len(canon)-3], ErrMalformed}, + {"unknown field", append(append([]byte(nil), canon...), 0xF8, 0x06, 0x01), ErrMalformed}, + {"duplicate version", mut(func(fs []rawField) []rawField { + return append([]rawField{fs[0]}, fs...) + }), ErrMalformed}, + {"out of order", mut(func(fs []rawField) []rawField { + fs[0], fs[1] = fs[1], fs[0] + return fs + }), ErrMalformed}, + {"version 2", mut(func(fs []rawField) []rawField { + fs[0].v = 2 + return fs + }), ErrUnsupportedVersion}, + {"version 0", mut(func(fs []rawField) []rawField { + fs[0].v = 0 + return fs + }), ErrUnsupportedVersion}, + {"unknown scheme", mut(func(fs []rawField) []rawField { + fs[1].v = 3 + return fs + }), ErrUnknownScheme}, + {"wrong wire type for scheme", mut(func(fs []rawField) []rawField { + fs[1] = rawField{num: fs[1].num, typ: protowire.BytesType, b: []byte{1}} + return fs + }), ErrMalformed}, + {"short release id", mut(func(fs []rawField) []rawField { + fs[2].b = fs[2].b[:31] + return fs + }), ErrInvalidField}, + {"non-minimal version varint", append([]byte{0x08, 0x81, 0x00}, canon[2:]...), ErrMalformed}, + {"missing title", mut(func(fs []rawField) []rawField { + return append(fs[:3], fs[4:]...) + }), ErrMalformed}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if _, err := Decode(tc.raw); !errors.Is(err, tc.want) { + t.Fatalf("Decode = %v, want %v", err, tc.want) + } + }) + } +} + +func TestDecodeRejectsOversized(t *testing.T) { + if _, err := Decode(make([]byte, MaxDecompressedSize+1)); !errors.Is(err, ErrTooLarge) { + t.Fatalf("got %v", err) + } +} + +func TestDecodeRejectsHostileCounts(t *testing.T) { + // A segment table claiming 2^60 positions must be refused before any + // allocation proportional to the claim. + canon := mustEncode(t, goodBundle(t)) + fields := splitTop(t, canon) + patched := false + for i, f := range fields { + if f.num != fFiles || patched { + continue + } + file := splitTop(t, f.b) + for j, ff := range file { + if ff.num == fSegs { + seg := splitTop(t, ff.b) + seg[0].v = 1 << 60 + file[j].b = joinTop(seg) + patched = true + } + } + fields[i].b = joinTop(file) + } + if !patched { + t.Fatal("no file with a segment table") + } + if _, err := Decode(joinTop(fields)); !errors.Is(err, ErrLimit) { + t.Fatalf("got %v, want ErrLimit", err) + } +} + +func TestComputeID(t *testing.T) { + raw := mustEncode(t, goodBundle(t)) + id := ComputeID(raw) + if id == (BundleID{}) || id != ComputeID(raw) { + t.Fatal("ComputeID unstable or zero") + } + raw[len(raw)-1] ^= 1 + if ComputeID(raw) == id { + t.Fatal("different bytes, same id") + } + if len(id.String()) != 64 { + t.Fatal("bad hex") + } +} diff --git a/internal/peernet/bundleimport/accept.go b/internal/peernet/bundleimport/accept.go new file mode 100644 index 000000000..24a8c4623 --- /dev/null +++ b/internal/peernet/bundleimport/accept.go @@ -0,0 +1,440 @@ +package bundleimport + +import ( + "context" + "errors" + "fmt" + "path" + "slices" + "strings" + "unicode" + + "golang.org/x/text/unicode/norm" + + "github.com/kipsilabs/altmount/internal/contentverify" + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/holes" + "github.com/kipsilabs/altmount/internal/importer/filesystem" + "github.com/kipsilabs/altmount/internal/importer/parser/fileinfo" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +// cleanTarget normalises the local target folder and refuses anything that is +// not a plain relative folder. +func cleanTarget(dir string) (string, error) { + dir = strings.Trim(strings.ReplaceAll(dir, `\`, "/"), "/") + if dir == "" { + return "", errors.New("empty target folder") + } + parts := strings.Split(dir, "/") + if len(parts) < 2 { + return "", errors.New("target folder must be below a category folder") + } + for _, part := range parts { + if part == "" || part == "." || part == ".." { + return "", fmt.Errorf("invalid target folder component %q", part) + } + for _, r := range filesystem.ReservedFolderNames { + if strings.EqualFold(part, r) { + return "", fmt.Errorf("reserved target folder component %q", part) + } + } + for _, r := range part { + if unicode.IsControl(r) { + return "", errors.New("control character in target folder") + } + } + } + return path.Clean(dir), nil +} + +func (i *Importer) rejected(r Reason, err error) (*Result, error) { + i.cnt.add(func(c *counters) { c.rejected[r]++ }) + return nil, reject(r, err) +} + +// Accept is the accept pipeline: re-validate, local policy, release probe, +// media probe through the layout in a hidden staging area, and only then the +// local write in pending state. Any rejection leaves nothing behind. +func (i *Importer) Accept(ctx context.Context, req Request) (*Result, error) { + ctx, cancel := context.WithTimeout(ctx, i.cfg.Policy.AcceptTimeout) + defer cancel() + b := req.Bundle + if b == nil { + return i.rejected(ReasonInvalid, errors.New("no bundle")) + } + if err := bundle.Validate(b); err != nil { + if errors.Is(err, bundle.ErrBadFilename) || errors.Is(err, bundle.ErrDuplicateName) { + return i.rejected(ReasonBadName, err) + } + return i.rejected(ReasonInvalid, err) + } + pol := i.cfg.Policy + switch { + case b.TotalSize > pol.MaxTotalSize: + return i.rejected(ReasonTooLarge, fmt.Errorf("%d bytes", b.TotalSize)) + case len(b.Files) > pol.MaxFiles: + return i.rejected(ReasonTooManyFiles, fmt.Errorf("%d files", len(b.Files))) + } + if i.cfg.HealthEnabled == nil || !i.cfg.HealthEnabled() { + return i.rejected(ReasonHealthDisabled, errors.New("the full check could never run")) + } + dir, err := cleanTarget(req.Target.Dir) + if err != nil { + return i.rejected(ReasonBadTarget, err) + } + loc, err := bundle.ToLocal(b, i.cfg.Now()) + if err != nil { + return i.rejected(ReasonInvalid, err) + } + if err := verifyIdentity(b, loc); err != nil { + return i.rejected(ReasonInvalid, err) + } + if req.ExpectedNZB != nil && (b.Scheme != bundle.SchemeNZB || !slices.EqualFunc(req.ExpectedNZB, loc.NZBFiles, slices.Equal[[]string])) { + return i.rejected(ReasonNZBMismatch, errors.New("segment list differs from the supplied NZB")) + } + + if err := checkSizes(b, req); err != nil { + return i.rejected(ReasonSizeImplausible, err) + } + keep, r, err := i.selectFiles(b) + if err != nil { + return i.rejected(r, err) + } + b, loc = subset(b, loc, keep) + + paths := make([]string, len(loc.Files)) + for n, lf := range loc.Files { + paths[n] = dir + "/" + lf.Name + } + existing, err := i.checkTargets(b, paths) + if err != nil { + return i.rejected(ReasonCollision, err) + } + sizes := make([]int64, len(loc.Files)) + for n, lf := range loc.Files { + sizes[n] = lf.Meta.FileSize + } + res := &Result{Title: b.Title, Dir: dir, Paths: paths, Sizes: sizes, TotalSize: b.TotalSize} + if existing == len(paths) { + res.AlreadyImported = true + i.cnt.add(func(c *counters) { c.already++ }) + return res, nil + } + + if i.cfg.Prober == nil { + return i.rejected(ReasonProbeInconclusive, errors.New("no release prober")) + } + if err := i.cfg.Prober.ProbeRelease(ctx, probeFiles(loc)); err != nil { + if errors.Is(err, ErrProbeMissing) { + return i.rejected(ReasonProbeMissing, err) + } + return i.rejected(ReasonProbeInconclusive, err) + } + if r, err := i.verifyYenc(ctx, b); err != nil { + if r == ReasonYencMismatch && i.cfg.Reporter != nil { + // The layout contradicts the articles themselves: the publisher + // served a bad bundle, which is attributable (fetch domain only). + i.cfg.Reporter.ReportBadBundle(ctx, req.Origin, b.ReleaseID.String()) + i.cnt.add(func(c *counters) { c.reports++ }) + } + return i.rejected(r, err) + } + if r, err := i.probeMedia(ctx, loc); err != nil { + return i.rejected(r, err) + } + + unlock := i.lockTarget(dir) + defer unlock() + // Re-check under the lock: another accept may have written meanwhile. + existing, err = i.checkTargets(b, paths) + if err != nil { + return i.rejected(ReasonCollision, err) + } + if existing == len(paths) { + res.AlreadyImported = true + i.cnt.add(func(c *counters) { c.already++ }) + return res, nil + } + if err := i.write(ctx, b, loc, req, res); err != nil { + return i.rejected(ReasonWriteFailed, err) + } + i.cnt.add(func(c *counters) { c.accepted++ }) + return res, nil +} + +// verifyIdentity recomputes every File ID and the Release ID from the layout +// the bundle describes, so this package does not depend on its caller having +// run bundle.Verify. +func verifyIdentity(b *bundle.Bundle, loc *bundle.Local) error { + ids := make([]contentid.ID, len(loc.Files)) + for n, lf := range loc.Files { + id, err := contentid.FileID(lf.Meta) + if err != nil { + return fmt.Errorf("file %d: %w", n, err) + } + if id != b.Files[n].FileID { + return bundle.ErrFileIDMismatch + } + ids[n] = id + } + var rid contentid.ID + if b.Scheme == bundle.SchemeFiles { + rid = bundle.FilesReleaseID(ids) + } else { + store := &metapb.NzbStore{} + for _, f := range loc.NZBFiles { + fe := &metapb.NzbFileEntry{} + for _, id := range f { + fe.Segments = append(fe.Segments, &metapb.NzbSeg{Id: id}) + } + store.Files = append(store.Files, fe) + } + var err error + if rid, err = contentid.ReleaseID(store); err != nil { + return err + } + } + if rid != b.ReleaseID { + return bundle.ErrReleaseIDMismatch + } + return nil +} + +// foldName is the comparison key of a file name: Unicode NFC, then lower case. +func foldName(n string) string { return strings.ToLower(norm.NFC.String(n)) } + +// checkTargets returns how many target paths already hold the very same file +// of the very same release. A path holding a different file, a name that only +// differs by case or normalisation form from one of ours, or only some paths +// holding theirs, is a collision. +func (i *Importer) checkTargets(b *bundle.Bundle, paths []string) (int, error) { + taken := map[string]string{} + if len(paths) > 0 { + names, err := i.cfg.Library.ListDirectory(path.Dir(paths[0])) + if err != nil { + return 0, fmt.Errorf("list target folder: %w", err) + } + for _, n := range names { + taken[foldName(n)] = n + } + } + same := 0 + for n, p := range paths { + base := path.Base(p) + if other, ok := taken[foldName(base)]; ok && other != base { + return 0, fmt.Errorf("%s: %q is already there under another spelling", p, other) + } + if !i.cfg.Library.FileExists(p) { + continue + } + m, err := i.cfg.Library.ReadFileMetadata(p) + if err != nil || m == nil { + return 0, fmt.Errorf("%s: unreadable existing file", p) + } + id, err := contentid.FileID(m) + if err != nil || id != b.Files[n].FileID { + return 0, fmt.Errorf("%s: a different file is already there", p) + } + if i.cfg.State != nil { + row, found, err := i.cfg.State.Get(context.Background(), p) + if err != nil { + return 0, fmt.Errorf("%s: %w", p, err) + } + if !found || row.ReleaseID != b.ReleaseID.String() { + return 0, fmt.Errorf("%s: the same file belongs to another release", p) + } + } + same++ + } + if same != 0 && same != len(paths) { + return 0, errors.New("only part of the release is already present") + } + return same, nil +} + +func realIDs(segs []*metapb.SegmentData, out []string) []string { + for _, s := range segs { + if s != nil && s.Id != "" && !holes.IsPlaceholderID(s.Id) { + out = append(out, s.Id) + } + } + return out +} + +func probeFiles(loc *bundle.Local) []ProbeFile { + out := make([]ProbeFile, 0, len(loc.Files)) + for _, lf := range loc.Files { + ids := realIDs(lf.Meta.SegmentData, nil) + for _, o := range lf.Meta.SharedOuterSources { + ids = realIDs(o.Segments, ids) + } + out = append(out, ProbeFile{Name: lf.Name, SegmentIDs: ids}) + } + return out +} + +// probeMedia reads the head of the main media file(s) through the real reader +// stack, in a hidden staging area that is removed on every outcome. +func (i *Importer) probeMedia(ctx context.Context, loc *bundle.Local) (Reason, error) { + var media []bundle.LocalFile + for _, lf := range loc.Files { + if fileinfo.IsVerifiableMediaFile(lf.Name) { + media = append(media, lf) + } + } + if len(media) == 0 { + return ReasonNoMediaFile, errors.New("no file the media probe can verify") + } + if i.cfg.Stager == nil { + return ReasonStaging, errors.New("no stager") + } + st, err := i.cfg.Stager.NewStaging() + if err != nil { + return ReasonStaging, err + } + defer func() { _ = st.Close() }() + for _, lf := range media { + if err := st.WriteFileMetadata(lf.Name, lf.Meta); err != nil { + return ReasonStaging, err + } + res := contentverify.Probe(ctx, st.Opener(), lf.Name, i.cfg.Policy.MediaTimeout()) + switch res.Result { + case contentverify.ContentValid: + case contentverify.ContentInvalid: + return ReasonMediaInvalid, fmt.Errorf("%s: %w", lf.Name, res.Err) + case contentverify.ContentSegmentMissing: + return ReasonMediaMissing, fmt.Errorf("%s: %w", lf.Name, res.Err) + default: + return ReasonMediaInconclusive, fmt.Errorf("%s: %w", lf.Name, res.Err) + } + if err := verifyStructure(ctx, st.Opener(), lf.Name, lf.Meta.FileSize); err != nil { + return structureReason(err), fmt.Errorf("%s: %w", lf.Name, err) + } + } + return "", nil +} + +// write is the only step that changes anything the user can see. The order is +// state rows (pending), then the metadata files, then the content index, so a +// crash can never leave a visible file without its pending row, and a failure, +// panic or cancellation after any step removes exactly what this call created +// (the caller holds the per-target lock, so nothing here belongs to anyone +// else). +func (i *Importer) write(ctx context.Context, b *bundle.Bundle, loc *bundle.Local, req Request, res *Result) (err error) { + var written []string + var rowsDone, indexDone bool + committed := false + defer func() { + if committed || i.crashed { + return + } + bg := context.WithoutCancel(ctx) + for _, p := range written { + _ = i.cfg.Library.DeleteFileMetadata(bg, p) + } + if rowsDone { + _ = i.cfg.State.Delete(bg, res.Paths) + } + if indexDone && i.cfg.Index != nil { + for _, p := range res.Paths { + _ = i.cfg.Index.Delete(bg, p) + } + } + }() + + rows := make([]database.PeernetBundleImport, 0, len(loc.Files)) + for n := range loc.Files { + rows = append(rows, database.PeernetBundleImport{ + FilePath: res.Paths[n], FileID: b.Files[n].FileID.String(), ReleaseID: b.ReleaseID.String(), + NetworkID: req.Origin.NetworkID, PublisherID: req.Origin.PublisherID, QueueID: req.Target.QueueID, + ArrOwed: !req.Target.SkipArr, Category: req.Target.Category, TargetPath: req.Target.TargetPath, + ResultingPath: res.Dir, + }) + } + rowsDone = true // set first: a failed batch may have inserted some rows + if err := i.cfg.State.Upsert(ctx, rows); err != nil { + return err + } + if i.crashAt("rows") { + return errCrashed + } + // A file the health system will never create a record for would stay + // pending forever and hold the whole import back: it is marked passed at + // once (the media files that do get checked decide the import). + if i.cfg.WillBeChecked != nil { + for n := range loc.Files { + if !i.cfg.WillBeChecked(res.Paths[n]) { + if _, err := i.cfg.State.MarkPassed(ctx, res.Paths[n]); err != nil { + return err + } + } + } + } + for n, lf := range loc.Files { + if loc.NZBPassword != "" { + lf.Meta.Password = loc.NZBPassword + } + if err := i.cfg.Library.WriteFileMetadata(res.Paths[n], lf.Meta); err != nil { + return err + } + written = append(written, res.Paths[n]) + if i.crashAt("file") { + return errCrashed + } + } + if i.crashAt("files") { + return errCrashed + } + // The verified release identity goes into the index after the files: the + // metadata observer may have computed a provisional entry meanwhile, and + // this one replaces it (nothing in the written metadata can reproduce the + // ID or the group). + if i.cfg.Index != nil { + scheme := database.PeernetSchemeNZB + if b.Scheme == bundle.SchemeFiles { + scheme = database.PeernetSchemeFiles + } + entries := make([]database.PeernetContentEntry, len(loc.Files)) + for n := range loc.Files { + entries[n] = contentid.VerifiedEntry(res.Paths[n], b.Files[n].FileID, b.ReleaseID, scheme) + } + indexDone = true + if err := i.cfg.Index.UpsertBatch(ctx, entries); err != nil { + return err + } + } + if i.crashAt("index") { + return errCrashed + } + // Last: only now is the import complete. A crash before this leaves rows + // that are not committed, which start-up rolls back whole. + if err := i.cfg.State.CommitRows(ctx, res.Paths); err != nil { + return err + } + committed = true + return nil +} + +// lockTarget serialises the check-and-write of one target folder. +func (i *Importer) lockTarget(dir string) func() { + i.tmu.Lock() + l := i.targets[dir] + if l == nil { + l = &targetLock{} + i.targets[dir] = l + } + l.refs++ + i.tmu.Unlock() + l.mu.Lock() + return func() { + l.mu.Unlock() + i.tmu.Lock() + if l.refs--; l.refs == 0 { + delete(i.targets, dir) + } + i.tmu.Unlock() + } +} diff --git a/internal/peernet/bundleimport/accept_test.go b/internal/peernet/bundleimport/accept_test.go new file mode 100644 index 000000000..311e8fac6 --- /dev/null +++ b/internal/peernet/bundleimport/accept_test.go @@ -0,0 +1,556 @@ +package bundleimport + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/javi11/nntppool/v5" + "github.com/kipsilabs/altmount/internal/config" + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/metadata" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/contentid" + "github.com/kipsilabs/altmount/internal/testsupport/fakepool" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + testKey = []byte("0123456789abcdef0123456789abcdef") + testIV = []byte("fedcba9876543210") +) + +type fakeProber struct { + err error + calls int + files []ProbeFile +} + +func (p *fakeProber) ProbeRelease(_ context.Context, files []ProbeFile) error { + p.calls++ + p.files = files + return p.err +} + +type fakeAnnouncer struct{ got []Announcement } + +func (a *fakeAnnouncer) AnnounceVerifiedImport(_ context.Context, an Announcement) error { + a.got = append(a.got, an) + return nil +} + +type fakeReporter struct{ got, verified []string } + +func (r *fakeReporter) ReportVerifiedBundle(_ context.Context, o Origin, release string) { + r.verified = append(r.verified, o.PublisherID+"/"+release) +} + +func (r *fakeReporter) ReportBadBundle(_ context.Context, o Origin, release string) { + r.got = append(r.got, o.PublisherID+"/"+release) +} + +type env struct { + t *testing.T + libRoot string + stageRoot string + lib *metadata.MetadataService + db *database.DB + repo *database.PeernetBundleImportRepository + fp *fakepool.Client + prober *fakeProber + ann *fakeAnnouncer + rep *fakeReporter + healthOn bool + imp *Importer +} + +func newEnv(t *testing.T) *env { + t.Helper() + e := &env{t: t, libRoot: t.TempDir(), stageRoot: t.TempDir(), fp: fakepool.New(), + prober: &fakeProber{}, ann: &fakeAnnouncer{}, rep: &fakeReporter{}, healthOn: true} + e.lib = metadata.NewMetadataService(e.libRoot) + db, err := database.NewDB(database.Config{DatabasePath: filepath.Join(t.TempDir(), "db")}) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + e.db = db + e.repo = database.NewPeernetBundleImportRepository(db.Connection(), database.DialectSQLite) + cfg := config.DefaultConfig() + stager := NewFSStager(FSStagerDeps{ + Pool: &fakePoolManager{client: e.fp}, + Config: func() *config.Config { return cfg }, + Root: e.stageRoot, + }) + e.imp = New(Config{Headers: NewPoolHeaderReader(&fakePoolManager{client: e.fp}), Library: e.lib, Stager: stager, Prober: e.prober, State: e.repo, Announcer: e.ann, + Reporter: e.rep, HealthEnabled: func() bool { return e.healthOn }}) + return e +} + +// fileSpec describes one file of a test bundle. The served articles hold plain +// (or its AES-CBC encryption under key); the bundle carries bundleKey. +type fileSpec struct { + name string + plain []byte + segSize int + key []byte + bundleKey []byte +} + +// mkvPlain builds a minimal but structurally valid Matroska file of size +// bytes: EBML header, a Segment whose size ends at the file end, a SeekHead +// pointing at Info and at Cues (the last element of the file), a Cluster of +// filler between them. +func mkvPlain(size int) []byte { + u32 := func(v int) []byte { return []byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)} } + seek := func(id []byte, pos int) []byte { + body := append(append([]byte{0x53, 0xAB, 0x84}, id...), append([]byte{0x53, 0xAC, 0x84}, u32(pos)...)...) + return append([]byte{0x4D, 0xBB, byte(0x80 | len(body))}, body...) + } + info := []byte{0x15, 0x49, 0xA9, 0x66} + cues := []byte{0x1C, 0x53, 0xBB, 0x6B} + ebml := append([]byte{0x1A, 0x45, 0xDF, 0xA3, 0x8B, 0x42, 0x82, 0x88}, "matroska"...) + segStart := len(ebml) + 12 + seekLen := 2 * 17 + seekHead := append([]byte{0x11, 0x4D, 0x9B, 0x74, byte(0x80 | seekLen)}, append(seek(info, seekLen+5), seek(cues, size-segStart-9)...)...) + out := append([]byte{}, ebml...) + segSize := size - segStart + out = append(out, 0x18, 0x53, 0x80, 0x67, 0x01, 0, byte(segSize>>40), byte(segSize>>32), byte(segSize>>24), byte(segSize>>16), byte(segSize>>8), byte(segSize)) + out = append(out, seekHead...) + out = append(out, append(append([]byte{}, info...), 0x84, 0, 0, 0, 0)...) + fill := size - len(out) - 6 - 9 + out = append(out, 0x1F, 0x43, 0xB6, 0x75, byte(0x40|fill>>8), byte(fill)) + for k := 0; k < fill; k++ { + out = append(out, byte((k*7+13)%251)) + } + out = append(out, append(append([]byte{}, cues...), 0x84, 0, 0, 0, 0)...) + return out +} + +func junkPlain(size int) []byte { + out := make([]byte, size) + for i := range out { + out[i] = byte(i%7 + 1) + } + return out +} + +func cbc(t *testing.T, key, plain []byte) []byte { + blk, err := aes.NewCipher(key) + require.NoError(t, err) + out := make([]byte, len(plain)) + cipher.NewCBCEncrypter(blk, testIV).CryptBlocks(out, plain) + return out +} + +// mkBundle serves the specs' articles from the fake pool and returns a valid +// nzb-scheme bundle (identifiers recomputed from what it describes). +func (e *env) mkBundle(tag string, specs ...fileSpec) *bundle.Bundle { + e.t.Helper() + b := &bundle.Bundle{Scheme: bundle.SchemeNZB, Title: "Test Release " + tag, PostedUnix: 1_700_000_000} + for fi, sp := range specs { + payload := sp.plain + var key, iv []byte + if sp.key != nil { + payload = cbc(e.t, sp.key, sp.plain) + key, iv = sp.key, testIV + if sp.bundleKey != nil { + key = sp.bundleKey + } + } + var segs []bundle.Segment + count := 0 + for i := 0; i*sp.segSize < len(payload); i++ { + end := min((i+1)*sp.segSize, len(payload)) + id := fmt.Sprintf("%s-%d-%03d@test", tag, fi, i) + e.fp.SetBehavior(id, fakepool.SegmentBehavior{Bytes: payload[i*sp.segSize : end], + YEnc: nntppool.YEncMeta{PartBegin: int64(i * sp.segSize), PartSize: int64(end - i*sp.segSize), FileSize: int64(len(payload))}}) + segs = append(segs, bundle.Segment{Ref: int64(len(b.Pool)), Size: int64(end - i*sp.segSize), Start: 0, End: int64(end - i*sp.segSize - 1)}) + b.Pool = append(b.Pool, id) + count++ + } + b.ReleaseFiles = append(b.ReleaseFiles, count) + b.Files = append(b.Files, bundle.File{Name: sp.name, Size: int64(len(sp.plain)), Segments: segs, AESKey: key, AESIV: iv}) + b.TotalSize += int64(len(sp.plain)) + } + for i := range b.Files { // provisional ascending ids so ToLocal validates + b.Files[i].FileID = contentid.ID{byte(i + 1)} + } + b.ReleaseID = contentid.ID{9} + loc, err := bundle.ToLocal(b, e.imp.cfg.Now()) + require.NoError(e.t, err) + for i := range b.Files { + id, err := contentid.FileID(loc.Files[i].Meta) + require.NoError(e.t, err) + b.Files[i].FileID = id + } + sort.Slice(b.Files, func(i, j int) bool { return string(b.Files[i].FileID[:]) < string(b.Files[j].FileID[:]) }) + store := &metapb.NzbStore{} + pos := 0 + for _, n := range b.ReleaseFiles { + fe := &metapb.NzbFileEntry{} + for _, id := range b.Pool[pos : pos+n] { + fe.Segments = append(fe.Segments, &metapb.NzbSeg{Id: id}) + } + pos += n + store.Files = append(store.Files, fe) + } + rid, err := contentid.ReleaseID(store) + require.NoError(e.t, err) + b.ReleaseID = rid + require.NoError(e.t, bundle.Validate(b)) + return b +} + +func (e *env) req(b *bundle.Bundle) Request { + return Request{Bundle: b, Origin: Origin{NetworkID: strings.Repeat("c", 64), PublisherID: strings.Repeat("d", 64)}, + Target: Target{Dir: "movies/Test.Release", Category: "movies", QueueID: 11}} +} + +func (e *env) filesOnDisk(root string) []string { + var out []string + _ = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err == nil && !d.IsDir() { + out = append(out, p) + } + return nil + }) + return out +} + +func (e *env) stateRows() int { + var n int + require.NoError(e.t, e.db.Connection().QueryRow(`SELECT COUNT(*) FROM peernet_bundle_imports`).Scan(&n)) + return n +} + +func (e *env) assertNothingBehind() { + e.t.Helper() + assert.Empty(e.t, e.filesOnDisk(e.libRoot), "no metadata file may be written") + entries, _ := os.ReadDir(e.stageRoot) + assert.Empty(e.t, entries, "no staging leftovers") + assert.Zero(e.t, e.stateRows(), "no state row") + var idx, q, h, fh int + conn := e.db.Connection() + _ = conn.QueryRow(`SELECT COUNT(*) FROM peernet_content_index`).Scan(&idx) + _ = conn.QueryRow(`SELECT COUNT(*) FROM import_queue`).Scan(&q) + _ = conn.QueryRow(`SELECT COUNT(*) FROM import_history`).Scan(&h) + _ = conn.QueryRow(`SELECT COUNT(*) FROM file_health`).Scan(&fh) + assert.Zero(e.t, idx+q+h+fh, "no index, queue, history or health row") +} + +func mkvSpec(name string) fileSpec { return fileSpec{name: name, plain: mkvPlain(512), segSize: 128} } + +func TestAccept_WritesPendingFilesAndMarksOrigin(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("ok", mkvSpec("movie.mkv")) + res, err := e.imp.Accept(context.Background(), e.req(b)) + require.NoError(t, err) + assert.Equal(t, []string{"movies/Test.Release/movie.mkv"}, res.Paths) + assert.Equal(t, "movies/Test.Release", res.Dir) + assert.False(t, res.AlreadyImported) + + meta, err := e.lib.ReadFileMetadata("movies/Test.Release/movie.mkv") + require.NoError(t, err) + require.NotNil(t, meta) + assert.Equal(t, int64(512), meta.FileSize) + + row, ok, err := e.repo.Get(context.Background(), "movies/Test.Release/movie.mkv") + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, database.PeernetBundleStatePending, row.State) + assert.Equal(t, strings.Repeat("d", 64), row.PublisherID) + assert.Equal(t, strings.Repeat("c", 64), row.NetworkID) + assert.Equal(t, b.ReleaseID.String(), row.ReleaseID) + assert.Equal(t, int64(11), row.QueueID) + assert.True(t, row.ArrOwed) + assert.Equal(t, uint64(1), e.imp.Counters().Accepted) + entries, _ := os.ReadDir(e.stageRoot) + assert.Empty(t, entries, "staging removed") +} + +func TestAccept_TargetComesFromTheRequestNeverTheBundle(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("tgt", mkvSpec("movie.mkv")) + b.Title = "movies Evil Title" // the title never steers the folder + req := e.req(b) + req.Target.Dir = "tv/Local.Choice" + res, err := e.imp.Accept(context.Background(), req) + require.NoError(t, err) + assert.Equal(t, []string{"tv/Local.Choice/movie.mkv"}, res.Paths) +} + +func TestAccept_StoresNZBPasswordOpaquelyOnTheFiles(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("pw", mkvSpec("movie.mkv")) + b.NZBPassword = "s3cret pass" + _, err := e.imp.Accept(context.Background(), e.req(b)) + require.NoError(t, err) + meta, err := e.lib.ReadFileMetadata("movies/Test.Release/movie.mkv") + require.NoError(t, err) + assert.Equal(t, "s3cret pass", meta.Password) + assert.Equal(t, metapb.Encryption_NONE, meta.Encryption, "the password is not used to decrypt anything") +} + +func TestAccept_RejectsBadBundles(t *testing.T) { + cases := map[string]struct { + mut func(e *env, b *bundle.Bundle, req *Request) + reason Reason + }{ + "nil bundle": {func(e *env, b *bundle.Bundle, r *Request) { r.Bundle = nil }, ReasonInvalid}, + "size mismatch": {func(e *env, b *bundle.Bundle, r *Request) { b.TotalSize++ }, ReasonInvalid}, + "path separator": {func(e *env, b *bundle.Bundle, r *Request) { b.Files[0].Name = "a/b.mkv" }, ReasonBadName}, + "dotdot name": {func(e *env, b *bundle.Bundle, r *Request) { b.Files[0].Name = ".." }, ReasonBadName}, + "empty target": {func(e *env, b *bundle.Bundle, r *Request) { r.Target.Dir = "" }, ReasonBadTarget}, + "traversal target": {func(e *env, b *bundle.Bundle, r *Request) { r.Target.Dir = "movies/../../x" }, ReasonBadTarget}, + "too large": {func(e *env, b *bundle.Bundle, r *Request) { + e.imp.cfg.Policy.MaxTotalSize = 100 + }, ReasonTooLarge}, + "too many files": {func(e *env, b *bundle.Bundle, r *Request) { e.imp.cfg.Policy.MaxFiles = 0 + 1 }, ReasonTooManyFiles}, + "health disabled": {func(e *env, b *bundle.Bundle, r *Request) { e.healthOn = false }, ReasonHealthDisabled}, + "release id wrong": {func(e *env, b *bundle.Bundle, r *Request) { + b.ReleaseID[0] ^= 0xff + }, ReasonInvalid}, + "file id wrong": {func(e *env, b *bundle.Bundle, r *Request) { b.Files[0].FileID[5] ^= 0xff }, ReasonInvalid}, + "nzb mismatch": {func(e *env, b *bundle.Bundle, r *Request) { + r.ExpectedNZB = [][]string{{"other@x"}} + }, ReasonNZBMismatch}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("bad", mkvSpec("movie.mkv"), mkvSpec("second.mkv")) + req := e.req(b) + tc.mut(e, b, &req) + _, err := e.imp.Accept(context.Background(), req) + require.Error(t, err) + assert.Equal(t, tc.reason, ReasonOf(err), "%v", err) + assert.Equal(t, uint64(1), e.imp.Counters().Rejected[tc.reason]) + e.assertNothingBehind() + assert.Empty(t, e.rep.got, "a local rejection never strikes anyone") + }) + } +} + +func TestAccept_NZBCrossCheckAcceptsTheSameList(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("same", mkvSpec("movie.mkv")) + req := e.req(b) + req.ExpectedNZB = [][]string{append([]string{}, b.Pool...)} + _, err := e.imp.Accept(context.Background(), req) + require.NoError(t, err) +} + +func TestAccept_CollisionWithDifferentContentIsRejected(t *testing.T) { + e := newEnv(t) + other := e.mkBundle("other", mkvSpec("movie.mkv")) + _, err := e.imp.Accept(context.Background(), e.req(other)) + require.NoError(t, err) + + b := e.mkBundle("mine", mkvSpec("movie.mkv"), mkvSpec("extra.mkv")) + _, err = e.imp.Accept(context.Background(), e.req(b)) + require.Error(t, err) + assert.Equal(t, ReasonCollision, ReasonOf(err)) + assert.False(t, e.lib.FileExists("movies/Test.Release/extra.mkv"), "nothing of the rejected bundle was written") + meta, err := e.lib.ReadFileMetadata("movies/Test.Release/movie.mkv") + require.NoError(t, err) + assert.Equal(t, other.Pool[0], meta.SegmentData[0].Id, "the existing file is untouched") +} + +func TestAccept_SameFileIDAtPathIsIdempotent(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("idem", mkvSpec("movie.mkv")) + ctx := context.Background() + _, err := e.imp.Accept(ctx, e.req(b)) + require.NoError(t, err) + _, err = e.repo.MarkPassed(ctx, "movies/Test.Release/movie.mkv") + require.NoError(t, err) + calls := e.prober.calls + + res, err := e.imp.Accept(ctx, e.req(b)) + require.NoError(t, err) + assert.True(t, res.AlreadyImported) + assert.Equal(t, calls, e.prober.calls, "no second probe") + row, _, _ := e.repo.Get(ctx, "movies/Test.Release/movie.mkv") + assert.Equal(t, database.PeernetBundleStatePassed, row.State, "a passed file is not reset to pending") + assert.Equal(t, uint64(1), e.imp.Counters().AlreadyImported) +} + +func TestAccept_ProbeFailureRejectsWithoutStrikeOrLeftovers(t *testing.T) { + for name, tc := range map[string]struct { + err error + reason Reason + }{ + "missing": {ErrProbeMissing, ReasonProbeMissing}, + "inconclusive": {ErrProbeInconclusive, ReasonProbeInconclusive}, + "other": {errors.New("boom"), ReasonProbeInconclusive}, + } { + t.Run(name, func(t *testing.T) { + e := newEnv(t) + e.prober.err = tc.err + b := e.mkBundle("probe", mkvSpec("movie.mkv")) + _, err := e.imp.Accept(context.Background(), e.req(b)) + require.Error(t, err) + assert.Equal(t, tc.reason, ReasonOf(err)) + e.assertNothingBehind() + assert.Empty(t, e.rep.got, "the publisher may simply have other providers: no strike") + }) + } +} + +func TestAccept_ProbeSeesEveryFilesSegments(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("pf", mkvSpec("movie.mkv"), mkvSpec("second.mkv")) + _, err := e.imp.Accept(context.Background(), e.req(b)) + require.NoError(t, err) + require.Len(t, e.prober.files, 2) + total := 0 + for _, f := range e.prober.files { + assert.Len(t, f.SegmentIDs, 4) + total += len(f.SegmentIDs) + } + assert.Equal(t, 8, total) +} + +func TestAccept_WrongAESKeyFailsMediaProbeAndLeavesNothing(t *testing.T) { + e := newEnv(t) + wrong := []byte("WRONGKEY9876543210WRONGKEY987654") + b := e.mkBundle("aes", fileSpec{name: "movie.mkv", plain: mkvPlain(512), segSize: 128, key: testKey, bundleKey: wrong}) + _, err := e.imp.Accept(context.Background(), e.req(b)) + require.Error(t, err) + assert.Equal(t, ReasonMediaInvalid, ReasonOf(err), "%v", err) + e.assertNothingBehind() + assert.Empty(t, e.rep.got) +} + +func TestAccept_RightAESKeyPassesMediaProbe(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("aesok", fileSpec{name: "movie.mkv", plain: mkvPlain(512), segSize: 128, key: testKey}) + _, err := e.imp.Accept(context.Background(), e.req(b)) + require.NoError(t, err) + assert.True(t, e.lib.FileExists("movies/Test.Release/movie.mkv")) +} + +func TestAccept_GarbageMediaFailsProbe(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("junk", fileSpec{name: "movie.mkv", plain: junkPlain(512), segSize: 128}) + _, err := e.imp.Accept(context.Background(), e.req(b)) + assert.Equal(t, ReasonMediaInvalid, ReasonOf(err)) + e.assertNothingBehind() +} + +func TestAccept_MissingHeadArticleFailsProbeAsMissing(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("head", mkvSpec("movie.mkv")) + e.fp.SetBehavior(b.Pool[0], fakepool.SegmentBehavior{Err: nntppool.ErrArticleNotFound}) + _, err := e.imp.Accept(context.Background(), e.req(b)) + // The first article is always among the yEnc samples, so its absence is + // now found there (before the media probe would find it). + assert.Equal(t, ReasonProbeMissing, ReasonOf(err), "%v", err) + e.assertNothingBehind() +} + +func TestAccept_NoVerifiableMediaFileIsRejected(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("doc", fileSpec{name: "readme.txt", plain: junkPlain(256), segSize: 128}) + _, err := e.imp.Accept(context.Background(), e.req(b)) + assert.Equal(t, ReasonNoMediaFile, ReasonOf(err)) + e.assertNothingBehind() +} + +func TestAccept_WriteFailureRollsBackEverything(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("wf", mkvSpec("movie.mkv"), mkvSpec("second.mkv")) + lib := &failingLibrary{Library: e.lib, failOn: 2} + e.imp.cfg.Library = lib + _, err := e.imp.Accept(context.Background(), e.req(b)) + require.Error(t, err) + assert.Equal(t, ReasonWriteFailed, ReasonOf(err)) + e.assertNothingBehind() +} + +type failingLibrary struct { + Library + failOn, n int +} + +func (l *failingLibrary) WriteFileMetadata(p string, m *metapb.FileMetadata) error { + l.n++ + if l.n == l.failOn { + return errors.New("disk full") + } + return l.Library.WriteFileMetadata(p, m) +} + +func TestAccept_RecordsTheVerifiedReleaseIdentityInTheContentIndex(t *testing.T) { + e := newEnv(t) + content := database.NewPeernetContentRepository(e.db.Connection(), database.DialectSQLite) + e.imp.cfg.Index = content + b := e.mkBundle("idx", mkvSpec("a.mkv"), mkvSpec("b.mkv")) + res, err := e.imp.Accept(context.Background(), e.req(b)) + require.NoError(t, err) + for _, p := range res.Paths { + row, ok, err := content.GetByPath(context.Background(), p) + require.NoError(t, err) + require.True(t, ok, p) + assert.Equal(t, b.ReleaseID.String(), row.ReleaseID) + assert.Equal(t, database.PeernetSchemeNZB, row.ReleaseScheme) + assert.NotEmpty(t, row.GroupKey) + assert.True(t, contentid.IsVerifiedBundleRow(row)) + meta, err := e.lib.ReadFileMetadata(p) + require.NoError(t, err) + fid, err := contentid.FileID(meta) + require.NoError(t, err) + assert.Equal(t, fid.String(), row.FileID) + } +} + +func TestAccept_FilesSchemeBundleIsIndexedAsFilesScheme(t *testing.T) { + e := newEnv(t) + content := database.NewPeernetContentRepository(e.db.Connection(), database.DialectSQLite) + e.imp.cfg.Index = content + b := e.mkBundle("idxf", mkvSpec("a.mkv")) + b.Scheme = bundle.SchemeFiles + b.ReleaseFiles = nil + ids := []contentid.ID{b.Files[0].FileID} + b.ReleaseID = bundle.FilesReleaseID(ids) + req := e.req(b) + _, err := e.imp.Accept(context.Background(), req) + require.NoError(t, err) + row, ok, err := content.GetByPath(context.Background(), "movies/Test.Release/a.mkv") + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, database.PeernetSchemeFiles, row.ReleaseScheme) + assert.Equal(t, b.ReleaseID.String(), row.ReleaseID) +} + +func TestAccept_WriteFailureLeavesNoIndexRows(t *testing.T) { + e := newEnv(t) + e.imp.cfg.Index = database.NewPeernetContentRepository(e.db.Connection(), database.DialectSQLite) + b := e.mkBundle("idxw", mkvSpec("movie.mkv"), mkvSpec("second.mkv")) + e.imp.cfg.Library = &failingLibrary{Library: e.lib, failOn: 2} + _, err := e.imp.Accept(context.Background(), e.req(b)) + require.Error(t, err) + e.assertNothingBehind() // includes the content index +} + +func TestAccept_ResultCarriesFileSizesInPathOrder(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("sz", fileSpec{name: "a.mkv", plain: mkvPlain(512), segSize: 128}, fileSpec{name: "b.mkv", plain: mkvPlain(256), segSize: 128}) + res, err := e.imp.Accept(context.Background(), e.req(b)) + require.NoError(t, err) + require.Len(t, res.Sizes, len(res.Paths)) + for n, p := range res.Paths { + meta, err := e.lib.ReadFileMetadata(p) + require.NoError(t, err) + assert.Equal(t, meta.FileSize, res.Sizes[n], p) + } +} diff --git a/internal/peernet/bundleimport/collision_test.go b/internal/peernet/bundleimport/collision_test.go new file mode 100644 index 000000000..e358f29cb --- /dev/null +++ b/internal/peernet/bundleimport/collision_test.go @@ -0,0 +1,169 @@ +package bundleimport + +import ( + "context" + "errors" + "path" + "strings" + "sync" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/database" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// exactLibrary behaves like a case- and normalisation-sensitive file system +// (the metadata service on the developer's machine may not): FileExists is an +// exact-name match against the real directory listing. +type exactLibrary struct{ Library } + +func (l exactLibrary) FileExists(p string) bool { + names, _ := l.ListDirectory(path.Dir(p)) + for _, n := range names { + if n == path.Base(p) { + return true + } + } + return false +} + +func TestAccept_CollisionIgnoresCaseAndNormalisation(t *testing.T) { + for name, pair := range map[string][2]string{ + "case": {"Movie.mkv", "movie.mkv"}, + "normalisation": {"café.mkv", "café.mkv"}, + } { + t.Run(name, func(t *testing.T) { + e := newEnv(t) + e.imp.cfg.Library = exactLibrary{e.lib} + first := e.mkBundle("first", mkvSpec(pair[0])) + _, err := e.imp.Accept(context.Background(), e.req(first)) + require.NoError(t, err) + + // Same slot, name differing only by case or normalisation form. + second := e.mkBundle("second", fileSpec{name: pair[1], plain: mkvPlain(512), segSize: 128}) + _, err = e.imp.Accept(context.Background(), e.req(second)) + require.Error(t, err) + assert.Equal(t, ReasonCollision, ReasonOf(err)) + }) + } +} + +func TestAccept_SameFileInAnotherReleaseIsACollision(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("rel-a", mkvSpec("movie.mkv")) + _, err := e.imp.Accept(context.Background(), e.req(b)) + require.NoError(t, err) + + // Same file layout (same File ID) offered under a different Release ID. + again := *b + again.ReleaseID[0] ^= 0xFF + req := e.req(&again) + _, err = e.imp.checkTargets(&again, []string{"movies/Test.Release/movie.mkv"}) + require.Error(t, err, "the same File ID is only idempotent when the Release ID also matches") + _ = req + assert.True(t, strings.Contains(err.Error(), "release"), err.Error()) +} + +// orderLibrary records, at the moment each metadata file becomes visible, +// whether its pending state row already existed and whether the index already +// held it. +type orderLibrary struct { + Library + e *env + rowsAt map[string]bool +} + +func (l *orderLibrary) WriteFileMetadata(p string, m *metapb.FileMetadata) error { + _, ok, _ := l.e.repo.Get(context.Background(), p) + l.rowsAt[p] = ok + return l.Library.WriteFileMetadata(p, m) +} + +func TestAccept_StateRowsExistBeforeAnyFileIsVisible(t *testing.T) { + e := newEnv(t) + lib := &orderLibrary{Library: e.lib, e: e, rowsAt: map[string]bool{}} + e.imp.cfg.Library = lib + b := e.mkBundle("ord", mkvSpec("a.mkv"), mkvSpec("b.mkv")) + _, err := e.imp.Accept(context.Background(), e.req(b)) + require.NoError(t, err) + require.Len(t, lib.rowsAt, 2) + for p, had := range lib.rowsAt { + assert.True(t, had, "%s was written before its pending row existed", p) + } +} + +type panicLibrary struct { + Library + n int +} + +func (l *panicLibrary) WriteFileMetadata(p string, m *metapb.FileMetadata) error { + if l.n++; l.n == 2 { + panic("boom") + } + return l.Library.WriteFileMetadata(p, m) +} + +func TestAccept_PanicWhileWritingLeavesNothingBehind(t *testing.T) { + e := newEnv(t) + e.imp.cfg.Library = &panicLibrary{Library: e.lib} + e.imp.cfg.Index = database.NewPeernetContentRepository(e.db.Connection(), database.DialectSQLite) + b := e.mkBundle("pan", mkvSpec("a.mkv"), mkvSpec("b.mkv")) + assert.Panics(t, func() { _, _ = e.imp.Accept(context.Background(), e.req(b)) }) + e.assertNothingBehind() +} + +type gateLibrary struct { + Library + mu sync.Mutex + writes int + entered chan struct{} + release chan struct{} +} + +func (l *gateLibrary) WriteFileMetadata(p string, m *metapb.FileMetadata) error { + l.mu.Lock() + l.writes++ + n := l.writes + l.mu.Unlock() + switch n { + case 1: + close(l.entered) + <-l.release + case 2: + return errors.New("second writer fails") + } + return l.Library.WriteFileMetadata(p, m) +} + +func TestAccept_ConcurrentAcceptsOfOneReleaseDoNotUndoEachOther(t *testing.T) { + e := newEnv(t) + idx := database.NewPeernetContentRepository(e.db.Connection(), database.DialectSQLite) + e.imp.cfg.Index = idx + lib := &gateLibrary{Library: e.lib, entered: make(chan struct{}), release: make(chan struct{})} + e.imp.cfg.Library = lib + b := e.mkBundle("conc", mkvSpec("a.mkv")) + req := e.req(b) + + errA := make(chan error, 1) + go func() { _, err := e.imp.Accept(context.Background(), req); errA <- err }() + <-lib.entered + errB := make(chan error, 1) + go func() { _, err := e.imp.Accept(context.Background(), req); errB <- err }() + time.Sleep(150 * time.Millisecond) + close(lib.release) + require.NoError(t, <-errA) + <-errB // B may be "already imported" or a clean rejection; it must not break A + + p := "movies/Test.Release/a.mkv" + assert.True(t, e.lib.FileExists(p), "A's file is still there") + _, ok, err := idx.GetByPath(context.Background(), p) + require.NoError(t, err) + assert.True(t, ok, "A's index entry is still there") + _, found, _ := e.repo.Get(context.Background(), p) + assert.True(t, found, "A's state row is still there") +} diff --git a/internal/peernet/bundleimport/failure_test.go b/internal/peernet/bundleimport/failure_test.go new file mode 100644 index 000000000..6f1496bf8 --- /dev/null +++ b/internal/peernet/bundleimport/failure_test.go @@ -0,0 +1,243 @@ +package bundleimport + +import ( + "context" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/database" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// failAnn is an Announcer that also ends failed imports, recording what the +// importer glue would be asked to do. RemoveFailedFile deletes the metadata +// like the real service does. +type failAnn struct { + fakeAnnouncer + lib interface { + DeleteFileMetadata(context.Context, string) error + } + mu sync.Mutex + failed map[int64]string + removed []string + freed []int64 +} + +func (a *failAnn) FailImport(_ context.Context, queueID int64, reason string) error { + a.mu.Lock() + defer a.mu.Unlock() + if a.failed == nil { + a.failed = map[int64]string{} + } + a.failed[queueID] = reason + return nil +} + +func (a *failAnn) RemoveFailedFile(ctx context.Context, p string) error { + a.mu.Lock() + a.removed = append(a.removed, p) + a.mu.Unlock() + return a.lib.DeleteFileMetadata(ctx, p) +} + +func (a *failAnn) ReleaseHold(_ context.Context, queueID int64) error { + a.mu.Lock() + a.freed = append(a.freed, queueID) + a.mu.Unlock() + return nil +} + +func (e *env) withFailAnn() *failAnn { + a := &failAnn{lib: e.lib} + e.imp.cfg.Announcer = a + e.imp.cfg.Index = database.NewPeernetContentRepository(e.db.Connection(), database.DialectSQLite) + return a +} + +func (e *env) rowCount(paths ...string) int { + n := 0 + for _, p := range paths { + if _, ok, _ := e.repo.Get(context.Background(), p); ok { + n++ + } + } + return n +} + +func TestFailure_FailedFullCheckFailsTheQueueItemAndRemovesTheWholeImport(t *testing.T) { + e := newEnv(t) + ann := e.withFailAnn() + _, res := e.accept("f", mkvSpec("a.mkv"), mkvSpec("b.mkv")) + ctx := context.Background() + // An unrelated, ordinary file next to it must survive. + require.NoError(t, e.lib.WriteFileMetadata("movies/Other/other.mkv", mustMeta(t, e, res.Paths[0]))) + + e.imp.FullCheckPassed(ctx, res.Paths[0]) + e.imp.FullCheckFailed(ctx, res.Paths[1]) + + assert.Equal(t, failReasonCheck, ann.failed[11], "the queue item is failed, so the ARR blocklists and searches again") + for _, p := range res.Paths { + assert.False(t, e.lib.FileExists(p), "%s must not stay in the library", p) + } + assert.True(t, e.lib.FileExists("movies/Other/other.mkv"), "only the failed import is removed") + assert.Zero(t, e.rowCount(res.Paths...), "state rows are gone") + // The sibling that passed earlier can no longer complete the import. + e.imp.FullCheckPassed(ctx, res.Paths[0]) + e.imp.ResumeAnnouncements(ctx) + assert.Empty(t, ann.got, "a failed import is never announced") +} + +func TestFailure_StaleImportIsFailedLikeAFailedCheck(t *testing.T) { + e := newEnv(t) + ann := e.withFailAnn() + _, res := e.accept("stale", mkvSpec("a.mkv")) + e.imp.cfg.Now = func() time.Time { return time.Now().Add(MaxPendingAge + time.Hour) } + + assert.Equal(t, 1, e.imp.EscalateStale(context.Background())) + + assert.Equal(t, failReasonStale, ann.failed[11]) + assert.False(t, e.lib.FileExists(res.Paths[0])) + assert.Zero(t, e.rowCount(res.Paths...)) + assert.Empty(t, e.rep.got, "no check failed, so the publisher is not reported") +} + +func TestFailure_OrphanedHoldIsReleased(t *testing.T) { + e := newEnv(t) + ann := e.withFailAnn() + ctx := context.Background() + // A held queue item (peer_hold set) that no row refers to. + repo := database.NewRepository(e.db.Connection(), database.DialectSQLite) + item := &database.ImportQueueItem{NzbPath: "x.nzb", Status: database.QueueStatusCompleted, Priority: database.QueuePriorityNormal} + require.NoError(t, repo.AddToQueue(ctx, item)) + require.NoError(t, e.repo.SetQueueHold(ctx, item.ID, true)) + + _, res := e.accept("orphan", mkvSpec("a.mkv")) // queue 11: a different item + obs := e.imp.PathObserver() + obs.FileDeleted(ctx, res.Paths[0]) + obs.Flush() + + assert.Equal(t, []int64{item.ID}, ann.freed, "a hold nothing can release is lifted") +} + +func TestReconcile_DropsRowsWhoseFilesAreGoneAndAnnouncesNothing(t *testing.T) { + e := newEnv(t) + ann := e.withFailAnn() + ctx := context.Background() + _, res := e.accept("gone", mkvSpec("a.mkv")) + _, err := e.repo.MarkPassed(ctx, res.Paths[0]) + require.NoError(t, err) + require.NoError(t, e.lib.DeleteFileMetadata(ctx, res.Paths[0])) + e.imp.PathObserver() // not installed: the file vanished behind our back + + e.imp.Reconcile(ctx) + e.imp.ResumeAnnouncements(ctx) + + assert.Zero(t, e.rowCount(res.Paths...)) + assert.Empty(t, ann.got, "a release with no files left is never announced") +} + +// A crash at every step of write: the rows exist uncommitted, a restart rolls +// the whole import back, and until then nothing can announce or publish it. +func TestReconcile_RollsBackAnImportThatCrashedAtEveryStepOfWrite(t *testing.T) { + for _, tc := range []struct { + step string + nth int + }{{"rows", 0}, {"file", 0}, {"file", 1}, {"files", 0}, {"index", 0}} { + t.Run(fmt.Sprintf("%s#%d", tc.step, tc.nth), func(t *testing.T) { + e := newEnv(t) + ann := e.withFailAnn() + ctx := context.Background() + seen := 0 + e.imp.crashHook = func(step string) bool { + if step != tc.step { + return false + } + seen++ + return seen == tc.nth+1 + } + b := e.mkBundle("crash", mkvSpec("a.mkv"), mkvSpec("b.mkv")) + _, err := e.imp.Accept(ctx, e.req(b)) + require.Error(t, err, "the simulated crash abandons the write") + paths := []string{"movies/Test.Release/a.mkv", "movies/Test.Release/b.mkv"} + require.Equal(t, 2, e.rowCount(paths...), "the crash left uncommitted rows behind") + + // Before the restart: even when everything looks passed, nothing is + // announced or publishable. + for _, p := range paths { + _, _ = e.repo.MarkPassed(ctx, p) + } + e.imp.ResumeAnnouncements(ctx) + assert.Empty(t, ann.got, "an unfinished import is never announced") + _, passed, _ := e.imp.PeerBundleStatus(ctx, paths[0]) + assert.False(t, passed, "nor publishable") + + // The restart: a new importer over the same state. + imp2 := New(Config{Library: e.lib, State: e.repo, Announcer: ann, Index: e.imp.cfg.Index}) + imp2.Reconcile(ctx) + + for _, p := range paths { + assert.False(t, e.lib.FileExists(p), "%s was rolled back", p) + } + assert.Zero(t, e.rowCount(paths...)) + imp2.ResumeAnnouncements(ctx) + assert.Empty(t, ann.got) + }) + } +} + +func TestReconcile_LeavesAFinishedImportAndAForeignFileAlone(t *testing.T) { + e := newEnv(t) + e.withFailAnn() + ctx := context.Background() + _, res := e.accept("keep", mkvSpec("a.mkv")) + + // An uncommitted row whose path holds somebody else's file: the row goes, + // the file stays. + foreign := "movies/Foreign/x.mkv" + m := mustMeta(t, e, res.Paths[0]) + m.FileSize++ // a different file + m.SegmentData[0].Id = "other-id" + require.NoError(t, e.lib.WriteFileMetadata(foreign, m)) + require.NoError(t, e.repo.Upsert(ctx, []database.PeernetBundleImport{{FilePath: foreign, FileID: hex("e"), ReleaseID: hex("f"), NetworkID: hex("c"), PublisherID: hex("d")}})) + + e.imp.Reconcile(ctx) + + assert.True(t, e.lib.FileExists(res.Paths[0]), "a committed import stays") + assert.Equal(t, 1, e.rowCount(res.Paths...)) + assert.True(t, e.lib.FileExists(foreign), "a file that is not the row's file is never deleted") + assert.Zero(t, e.rowCount(foreign)) +} + +func hex(c string) string { return strings.Repeat(c, 64) } + +func mustMeta(t *testing.T, e *env, path string) *metapb.FileMetadata { + t.Helper() + m, err := e.lib.ReadFileMetadata(path) + require.NoError(t, err) + require.NotNil(t, m) + return m +} + +// An import no ARR is told about (suppressed by the caller, or a direct +// import) still ends its hold when the last file passes. +func TestFailure_HoldEndsWhenAnImportNoARRKnowsPasses(t *testing.T) { + e := newEnv(t) + ann := e.withFailAnn() + b := e.mkBundle("noarr", mkvSpec("a.mkv"), mkvSpec("b.mkv")) + req := e.req(b) + req.Target.SkipArr = true + res, err := e.imp.Accept(context.Background(), req) + require.NoError(t, err) + ctx := context.Background() + + e.imp.FullCheckPassed(ctx, res.Paths[0]) + assert.Empty(t, ann.freed, "one file still pending") + e.imp.FullCheckPassed(ctx, res.Paths[1]) + assert.Equal(t, []int64{11}, ann.freed) + assert.Empty(t, ann.got, "and no ARR is told") +} diff --git a/internal/peernet/bundleimport/filepolicy.go b/internal/peernet/bundleimport/filepolicy.go new file mode 100644 index 000000000..45171a7c1 --- /dev/null +++ b/internal/peernet/bundleimport/filepolicy.go @@ -0,0 +1,135 @@ +package bundleimport + +import ( + "fmt" + "path" + "slices" + "strings" + + "github.com/kipsilabs/altmount/internal/importer/parser/fileinfo" + "github.com/kipsilabs/altmount/internal/importer/utils" + "github.com/kipsilabs/altmount/internal/peernet/bundle" +) + +// ForbiddenExtensions are file types a media server or the operating system +// could act on (playlists, links, scripts, executables, queue files). A peer +// bundle containing one is refused whatever the allowed-extensions setting +// says. +var ForbiddenExtensions = []string{ + ".strm", ".nzb", ".nzbz", ".meta", ".url", ".lnk", ".m3u", ".m3u8", ".pls", ".desktop", + ".exe", ".com", ".scr", ".msi", ".dll", ".bat", ".cmd", ".ps1", ".vbs", ".js", ".jse", ".wsf", ".hta", + ".sh", ".bash", ".zsh", ".command", ".app", ".jar", ".py", ".pl", ".rb", ".php", ".apk", ".dmg", ".pkg", ".deb", ".rpm", + ".cpl", ".reg", ".inf", ".webloc", +} + +var subtitleExtensions = []string{".srt", ".ass", ".sub", ".idx", ".sup"} + +// fileKind says what a bundle file is for the local policy. +type fileKind int + +const ( + kindOther fileKind = iota // not written + kindMedia // verified media, written + kindSidecar // subtitle or nfo, written under a size cap +) + +func ext(name string) string { return strings.ToLower(path.Ext(name)) } + +func classify(name string) fileKind { + e := ext(name) + switch { + case fileinfo.IsVerifiableMediaFile(name): + return kindMedia + case e == ".nfo" || slices.Contains(subtitleExtensions, e): + return kindSidecar + } + return kindOther +} + +// selectFiles applies the local file policy to the bundle's files and returns +// the indices to write, in bundle order. Forbidden types reject the bundle; a +// file the importer's allowed-extensions list excludes, and any file that is +// neither verifiable media nor a small sidecar, is simply not written. +func (i *Importer) selectFiles(b *bundle.Bundle) ([]int, Reason, error) { + var allowed []string + if i.cfg.AllowedExtensions != nil { + allowed = i.cfg.AllowedExtensions() + } + pol := i.cfg.Policy + var keep []int + media, sidecars := 0, 0 + for n := range b.Files { + f := &b.Files[n] + if slices.Contains(ForbiddenExtensions, ext(f.Name)) { + return nil, ReasonForbiddenFile, fmt.Errorf("file type %q is never accepted from a peer", ext(f.Name)) + } + if !utils.IsAllowedFile(f.Name, f.Size, allowed, false) { + continue + } + switch classify(f.Name) { + case kindMedia: + if media++; media > pol.MediaFiles() { + return nil, ReasonTooManyFiles, fmt.Errorf("more than %d media files", pol.MediaFiles()) + } + case kindSidecar: + limit := pol.SubtitleBytes() + if ext(f.Name) == ".nfo" { + limit = pol.NfoBytes() + } + if f.Size > limit { + return nil, ReasonSidecarTooLarge, fmt.Errorf("%s is %d bytes", f.Name, f.Size) + } + if sidecars++; sidecars > pol.Sidecars() { + return nil, ReasonTooManyFiles, fmt.Errorf("more than %d sidecar files", pol.Sidecars()) + } + default: + continue + } + keep = append(keep, n) + } + if media == 0 { + return nil, ReasonNoMediaFile, fmt.Errorf("no media file to write") + } + return keep, "", nil +} + +// MediaFiles, Sidecars, SubtitleBytes and NfoBytes return the caps with their +// defaults for unset values. +func (p Policy) MediaFiles() int { return defaultInt(p.MaxMediaFiles, DefaultPolicy().MaxMediaFiles) } + +func (p Policy) Sidecars() int { return defaultInt(p.MaxSidecars, DefaultPolicy().MaxSidecars) } + +func (p Policy) SubtitleBytes() int64 { + if p.MaxSubtitleBytes <= 0 { + return DefaultPolicy().MaxSubtitleBytes + } + return p.MaxSubtitleBytes +} + +func (p Policy) NfoBytes() int64 { + if p.MaxNfoBytes <= 0 { + return DefaultPolicy().MaxNfoBytes + } + return p.MaxNfoBytes +} + +func defaultInt(v, def int) int { + if v <= 0 { + return def + } + return v +} + +// subset returns a copy of b and loc holding only the files at idx. Identity +// (Release ID, File IDs) was verified over the full bundle before this. +func subset(b *bundle.Bundle, loc *bundle.Local, idx []int) (*bundle.Bundle, *bundle.Local) { + nb, nl := *b, *loc + nb.Files = make([]bundle.File, len(idx)) + nl.Files = make([]bundle.LocalFile, len(idx)) + nb.TotalSize = 0 + for k, n := range idx { + nb.Files[k], nl.Files[k] = b.Files[n], loc.Files[n] + nb.TotalSize += b.Files[n].Size + } + return &nb, &nl +} diff --git a/internal/peernet/bundleimport/filepolicy_test.go b/internal/peernet/bundleimport/filepolicy_test.go new file mode 100644 index 000000000..23761d5d0 --- /dev/null +++ b/internal/peernet/bundleimport/filepolicy_test.go @@ -0,0 +1,90 @@ +package bundleimport + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAccept_RefusesFilesAMediaServerCouldAct(t *testing.T) { + for _, name := range []string{"x.strm", "x.nzb", "x.NZBZ", "x.meta", "x.url", "x.lnk", "x.m3u", "x.m3u8", "x.pls", "x.desktop", "x.exe", "x.sh", "x.bat", "x.cmd", "x.ps1", "x.py", "x.jar"} { + e := newEnv(t) + b := e.mkBundle("fb", mkvSpec("movie.mkv"), fileSpec{name: name, plain: junkPlain(200), segSize: 128}) + _, err := e.imp.Accept(context.Background(), e.req(b)) + require.Error(t, err, name) + assert.Equal(t, ReasonForbiddenFile, ReasonOf(err), name) + e.assertNothingBehind() + } +} + +func TestAccept_EveryWrittenMediaFileIsVerifiedNotOnlyTheLargest(t *testing.T) { + e := newEnv(t) + specs := []fileSpec{} + for i := 0; i < 5; i++ { + specs = append(specs, mkvSpec(fmt.Sprintf("good%d.mkv", i))) + } + // Smaller than the others, so a "probe the four largest" policy skips it. + specs = append(specs, fileSpec{name: "bad.mkv", plain: junkPlain(300), segSize: 128}) + b := e.mkBundle("pm", specs...) + _, err := e.imp.Accept(context.Background(), e.req(b)) + require.Error(t, err) + assert.Equal(t, ReasonMediaInvalid, ReasonOf(err)) + e.assertNothingBehind() +} + +func TestAccept_ExtensionAllowListAndSidecarBounds(t *testing.T) { + t.Run("allow list skips what the importer would skip", func(t *testing.T) { + e := newEnv(t) + e.imp.cfg.AllowedExtensions = func() []string { return []string{".mkv"} } + b := e.mkBundle("al", mkvSpec("movie.mkv"), fileSpec{name: "movie.srt", plain: junkPlain(100), segSize: 128}) + res, err := e.imp.Accept(context.Background(), e.req(b)) + require.NoError(t, err) + assert.Equal(t, []string{"movies/Test.Release/movie.mkv"}, res.Paths) + assert.False(t, e.lib.FileExists("movies/Test.Release/movie.srt")) + }) + t.Run("too many sidecars", func(t *testing.T) { + e := newEnv(t) + e.imp.cfg.Policy.MaxSidecars = 2 + specs := []fileSpec{mkvSpec("movie.mkv")} + for i := 0; i < 3; i++ { + specs = append(specs, fileSpec{name: fmt.Sprintf("s%d.srt", i), plain: junkPlain(100), segSize: 128}) + } + _, err := e.imp.Accept(context.Background(), e.req(e.mkBundle("sc", specs...))) + require.Error(t, err) + assert.Equal(t, ReasonTooManyFiles, ReasonOf(err)) + }) + t.Run("an oversized nfo", func(t *testing.T) { + e := newEnv(t) + e.imp.cfg.Policy.MaxNfoBytes = 50 + _, err := e.imp.Accept(context.Background(), e.req(e.mkBundle("nf", mkvSpec("movie.mkv"), fileSpec{name: "movie.nfo", plain: junkPlain(100), segSize: 128}))) + require.Error(t, err) + assert.Equal(t, ReasonSidecarTooLarge, ReasonOf(err)) + }) + t.Run("other files are not written", func(t *testing.T) { + e := newEnv(t) + res, err := e.imp.Accept(context.Background(), e.req(e.mkBundle("ot", mkvSpec("movie.mkv"), fileSpec{name: "readme.txt", plain: junkPlain(100), segSize: 128}))) + require.NoError(t, err) + assert.Len(t, res.Paths, 1) + }) +} + +func TestAccept_FilesThatWillNeverBeHealthCheckedAreNotLeftPending(t *testing.T) { + e := newEnv(t) + e.imp.cfg.WillBeChecked = func(p string) bool { return !strings.HasSuffix(p, ".srt") } + b := e.mkBundle("nc", mkvSpec("movie.mkv"), fileSpec{name: "movie.srt", plain: junkPlain(100), segSize: 128}) + res, err := e.imp.Accept(context.Background(), e.req(b)) + require.NoError(t, err) + var srt string + for _, p := range res.Paths { + if strings.HasSuffix(p, ".srt") { + srt = p + } + } + pending := e.imp.PendingFullCheck(context.Background(), res.Paths) + assert.NotContains(t, pending, srt, "a sidecar no health record is ever created for must not stay pending") + assert.Len(t, pending, 1, "the media file is still pending") +} diff --git a/internal/peernet/bundleimport/helpers_pool_test.go b/internal/peernet/bundleimport/helpers_pool_test.go new file mode 100644 index 000000000..f63ce9015 --- /dev/null +++ b/internal/peernet/bundleimport/helpers_pool_test.go @@ -0,0 +1,29 @@ +package bundleimport + +import ( + "context" + + "github.com/kipsilabs/altmount/internal/pool" +) + +// fakePoolManager serves one client; the embedded interface is nil, so a +// method the stack should not call panics instead of silently passing. +type fakePoolManager struct { + pool.Manager + client pool.NntpClient +} + +func (m *fakePoolManager) GetPool() (pool.NntpClient, error) { return m.client, nil } +func (m *fakePoolManager) HasPool() bool { return true } +func (m *fakePoolManager) IncArticlesDownloaded() {} +func (m *fakePoolManager) UpdateDownloadProgress(_ string, _ int64) {} +func (m *fakePoolManager) SpeculativeBudget() *pool.SpeculativeBudget { return nil } +func (m *fakePoolManager) NotifyStreamChange() {} +func (m *fakePoolManager) SetStreamSource(_ pool.StreamActivitySource) {} +func (m *fakePoolManager) AcquireImportSlot(_ context.Context) (func(), error) { + return func() {}, nil +} +func (m *fakePoolManager) AcquireImportConnection(_ context.Context) (func(), error) { + return func() {}, nil +} +func (m *fakePoolManager) StatSweepConcurrency(c int) int { return c } diff --git a/internal/peernet/bundleimport/importer.go b/internal/peernet/bundleimport/importer.go new file mode 100644 index 000000000..e962a95de --- /dev/null +++ b/internal/peernet/bundleimport/importer.go @@ -0,0 +1,83 @@ +package bundleimport + +import ( + "errors" + "golang.org/x/sync/singleflight" + "sync" + "time" +) + +// Config holds the collaborators of an Importer. Fetcher, Recorder, Announcer +// and Reporter may be nil: the matching step is then skipped (no fetch, no +// direct import, nothing announced, nothing reported). +type Config struct { + Library Library + Stager Stager + Prober ReleaseProber + State State + // Index, when set, receives the verified release identity of the files. + Index ContentIndex + Fetcher Fetcher + // Headers reads sampled articles' yEnc headers before anything is written. + Headers SegmentHeaders + Recorder Recorder + Announcer Announcer + Reporter BadBundleReporter + HealthEnabled func() bool + // AllowedExtensions returns the importer's allowed file extensions + // (config import.allowed_file_extensions); empty allows everything. + AllowedExtensions func() []string + // WillBeChecked says whether the health system will ever create a health + // record (and so a full check) for a written path under the current import + // strategy; nil means yes for every path. + WillBeChecked func(path string) bool + Policy Policy + Now func() time.Time +} + +// Importer accepts bundles. It is safe for concurrent use. +type Importer struct { + cfg Config + cnt counters + reported map[string]bool + // reportedMu guards reported. + reportedMu sync.Mutex + flight singleflight.Group + + tmu sync.Mutex + targets map[string]*targetLock + + // crashHook and crashed are a test seam: a hook that returns true at a + // step of write abandons it without any undo, as a process death would. + crashHook func(step string) bool + crashed bool +} + +var errCrashed = errors.New("bundleimport: simulated crash") + +func (i *Importer) crashAt(step string) bool { + if i.crashHook == nil || !i.crashHook(step) { + return false + } + i.crashed = true + return true +} + +type targetLock struct { + mu sync.Mutex + refs int +} + +// New builds an Importer. A zero Policy means DefaultPolicy. +func New(cfg Config) *Importer { + if cfg.Policy == (Policy{}) { + cfg.Policy = DefaultPolicy() + } + if cfg.Now == nil { + cfg.Now = time.Now + } + return &Importer{cfg: cfg, cnt: counters{rejected: map[Reason]uint64{}}, targets: map[string]*targetLock{}} +} + +// Counters returns a snapshot of the counters. +func (i *Importer) Counters() Counters { return i.cnt.snapshot() } diff --git a/internal/peernet/bundleimport/pathobserver.go b/internal/peernet/bundleimport/pathobserver.go new file mode 100644 index 000000000..c0bac0f8c --- /dev/null +++ b/internal/peernet/bundleimport/pathobserver.go @@ -0,0 +1,138 @@ +package bundleimport + +import ( + "context" + "log/slog" + "sync" + "time" + + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +// renameGrace is how long a deleted file's row waits for the matching write +// (the metadata service reports a rename as a delete followed by a write). +const renameGrace = 3 * time.Second + +// PathObserver keeps the peer-bundle state consistent with the library when +// the user deletes, moves or renames a file or folder: a deleted file loses its +// row (so a release with no files left is never announced), and a renamed file +// or folder keeps its row, state included, so a rename cannot lift the hold. +// It implements metadata.FileObserver and is meant to be a secondary observer. +type PathObserver struct { + i *Importer + + mu sync.Mutex + stash map[string]stashedRow // by File ID +} + +type stashedRow struct { + path string + timer *time.Timer +} + +// PathObserver returns the observer of this importer. +func (i *Importer) PathObserver() *PathObserver { + return &PathObserver{i: i, stash: map[string]stashedRow{}} +} + +// FileWritten re-attaches a row to a file that appeared under a new name. +func (o *PathObserver) FileWritten(ctx context.Context, path string) { + o.mu.Lock() + n := len(o.stash) + o.mu.Unlock() + if n == 0 || o.i.cfg.State == nil { + return + } + m, err := o.i.cfg.Library.ReadFileMetadata(path) + if err != nil || m == nil { + return + } + id, err := contentid.FileID(m) + if err != nil { + return + } + o.mu.Lock() + s, ok := o.stash[id.String()] + if ok { + s.timer.Stop() + delete(o.stash, id.String()) + } + o.mu.Unlock() + if !ok || s.path == path { + return + } + if _, err := o.i.cfg.State.MoveFile(ctx, s.path, path); err != nil { + slog.WarnContext(ctx, "peer bundle state could not follow a renamed file", "error", err) + } +} + +// FileDeleted holds the row for a moment: a rename arrives as delete + write. +func (o *PathObserver) FileDeleted(ctx context.Context, path string) { + if o.i.cfg.State == nil { + return + } + row, ok, err := o.i.cfg.State.Get(ctx, path) + if err != nil || !ok { + return + } + o.mu.Lock() + defer o.mu.Unlock() + if old, dup := o.stash[row.FileID]; dup { + old.timer.Stop() + } + bg := context.WithoutCancel(ctx) + o.stash[row.FileID] = stashedRow{path: path, timer: time.AfterFunc(renameGrace, func() { o.expire(bg, row.FileID, path) })} +} + +func (o *PathObserver) expire(ctx context.Context, fileID, path string) { + o.mu.Lock() + s, ok := o.stash[fileID] + if ok && s.path == path { + delete(o.stash, fileID) + } else { + ok = false + } + o.mu.Unlock() + if ok { + _ = o.i.cfg.State.Delete(ctx, []string{path}) + o.i.releaseOrphanHolds(ctx) + } +} + +// Flush settles every held row now (rows nobody claimed are deleted). Tests and +// shutdown use it. +func (o *PathObserver) Flush() { + o.mu.Lock() + pending := o.stash + o.stash = map[string]stashedRow{} + o.mu.Unlock() + for _, s := range pending { + s.timer.Stop() + _ = o.i.cfg.State.Delete(context.Background(), []string{s.path}) + } + if len(pending) > 0 { + o.i.releaseOrphanHolds(context.Background()) + } +} + +// DirectoryDeleted removes the rows below a deleted folder. +func (o *PathObserver) DirectoryDeleted(ctx context.Context, path string) { + if o.i.cfg.State == nil { + return + } + if err := o.i.cfg.State.DeleteDir(ctx, path); err != nil { + slog.WarnContext(ctx, "peer bundle state could not drop a deleted folder", "error", err) + return + } + o.i.releaseOrphanHolds(ctx) +} + +// DirectoryRenamed moves the rows below a folder with it. +func (o *PathObserver) DirectoryRenamed(ctx context.Context, from, to string) { + if o.i.cfg.State == nil { + return + } + if err := o.i.cfg.State.MoveDir(ctx, from, to); err != nil { + slog.WarnContext(ctx, "peer bundle state could not follow a renamed folder", "error", err) + } +} diff --git a/internal/peernet/bundleimport/pathobserver_test.go b/internal/peernet/bundleimport/pathobserver_test.go new file mode 100644 index 000000000..93b7e9621 --- /dev/null +++ b/internal/peernet/bundleimport/pathobserver_test.go @@ -0,0 +1,69 @@ +package bundleimport + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newObserved(t *testing.T) (*env, *Result) { + e := newEnv(t) + _, res := e.accept("po", mkvSpec("a.mkv"), mkvSpec("b.mkv")) + return e, res +} + +func TestPathObserver_DeletedFileRemovesItsRowAndNeverAnnouncesAnEmptyRelease(t *testing.T) { + e, res := newObserved(t) + ctx := context.Background() + obs := e.imp.PathObserver() + require.NoError(t, e.lib.DeleteFileMetadata(ctx, res.Paths[0])) + obs.FileDeleted(ctx, res.Paths[0]) + obs.Flush() + _, ok, _ := e.repo.Get(ctx, res.Paths[0]) + assert.False(t, ok, "the deleted file's row is gone") + _, ok, _ = e.repo.Get(ctx, res.Paths[1]) + assert.True(t, ok, "the other file keeps its row") + + // Deleting the last file leaves no row, so nothing can be announced. + require.NoError(t, e.lib.DeleteFileMetadata(ctx, res.Paths[1])) + obs.FileDeleted(ctx, res.Paths[1]) + obs.Flush() + assert.Zero(t, e.stateRows()) + e.imp.FullCheckPassed(ctx, res.Paths[1]) + assert.Empty(t, e.ann.got, "no announcement for a release with zero remaining files") +} + +func TestPathObserver_RenamedFileKeepsItsPendingRow(t *testing.T) { + e, res := newObserved(t) + ctx := context.Background() + obs := e.imp.PathObserver() + newPath := "movies/Test.Release/renamed.mkv" + require.NoError(t, e.lib.RenameFileMetadata(res.Paths[0], newPath)) + obs.FileDeleted(ctx, res.Paths[0]) + obs.FileWritten(ctx, newPath) + obs.Flush() + _, ok, _ := e.repo.Get(ctx, res.Paths[0]) + assert.False(t, ok) + row, ok, _ := e.repo.Get(ctx, newPath) + require.True(t, ok, "the row follows the renamed file") + assert.Equal(t, "pending", row.State, "a rename must not lift the hold") + assert.Contains(t, e.imp.PendingFullCheck(ctx, []string{newPath}), newPath) +} + +func TestPathObserver_RenamedFolderMovesRows(t *testing.T) { + e, res := newObserved(t) + ctx := context.Background() + obs := e.imp.PathObserver() + require.NoError(t, e.lib.RenameDirectory("movies/Test.Release", "movies/Other")) + obs.DirectoryRenamed(ctx, "movies/Test.Release", "movies/Other") + _, ok, _ := e.repo.Get(ctx, res.Paths[0]) + assert.False(t, ok) + row, ok, _ := e.repo.Get(ctx, "movies/Other/a.mkv") + require.True(t, ok) + assert.Equal(t, "movies/Other", row.ResultingPath) + + obs.DirectoryDeleted(ctx, "movies/Other") + assert.Zero(t, e.stateRows()) +} diff --git a/internal/peernet/bundleimport/reconcile.go b/internal/peernet/bundleimport/reconcile.go new file mode 100644 index 000000000..8aebcc37c --- /dev/null +++ b/internal/peernet/bundleimport/reconcile.go @@ -0,0 +1,107 @@ +package bundleimport + +import ( + "context" + "log/slog" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +// reconcilePage is how many state rows one page of the start-up walk reads. +const reconcilePage = 500 + +// Reconcile makes the peer-import state match the library after a restart, +// and is safe to run at any time: +// +// - A write that never finished (rows not committed: the process died +// between the first row and the last step of Importer.write) is rolled back +// whole: the files it wrote, its content-index entries and its rows are +// removed, so a partially written release is never announced, published or +// served. A file is removed only when it is the file the row describes. +// - A row whose file no longer exists is dropped. A release left with no row +// fires no announcement. +// - A held queue item that no row refers to any more is released. +func (i *Importer) Reconcile(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + slog.ErrorContext(ctx, "peer import reconcile stopped", "panic", r) + } + }() + if i.cfg.State == nil { + return + } + i.rollbackUncommitted(ctx) + i.dropRowsWithoutFiles(ctx) + i.releaseOrphanHolds(ctx) +} + +func (i *Importer) rollbackUncommitted(ctx context.Context) { + rows, err := i.cfg.State.ListUncommitted(ctx, 5000) + if err != nil { + slog.WarnContext(ctx, "could not list unfinished peer imports", "error", err) + return + } + if len(rows) == 0 { + return + } + paths := make([]string, 0, len(rows)) + for _, r := range rows { + paths = append(paths, r.FilePath) + if i.ownsFile(r) { + if err := i.cfg.Library.DeleteFileMetadata(ctx, r.FilePath); err != nil { + slog.WarnContext(ctx, "could not remove a file of an unfinished peer import", "file_path", r.FilePath, "error", err) + } + } + if i.cfg.Index != nil { + _ = i.cfg.Index.Delete(ctx, r.FilePath) + } + } + if err := i.cfg.State.Delete(ctx, paths); err != nil { + slog.WarnContext(ctx, "could not drop the rows of an unfinished peer import", "error", err) + return + } + slog.WarnContext(ctx, "rolled back peer imports that a restart interrupted", "files", len(paths)) +} + +// ownsFile reports whether the file at the row's path is the one the row +// describes (a missing file is not: nothing to remove). +func (i *Importer) ownsFile(r database.PeernetBundleImport) bool { + if !i.cfg.Library.FileExists(r.FilePath) { + return false + } + m, err := i.cfg.Library.ReadFileMetadata(r.FilePath) + if err != nil || m == nil { + // Unreadable: it can only be the half-written file of this import. + return true + } + id, err := contentid.FileID(m) + return err == nil && id.String() == r.FileID +} + +func (i *Importer) dropRowsWithoutFiles(ctx context.Context) { + var gone []string + after := "" + for { + rows, err := i.cfg.State.ListAfter(ctx, after, reconcilePage) + if err != nil { + slog.WarnContext(ctx, "could not walk the peer import state", "error", err) + return + } + for _, r := range rows { + if !i.cfg.Library.FileExists(r.FilePath) { + gone = append(gone, r.FilePath) + } + } + if len(rows) < reconcilePage { + break + } + after = rows[len(rows)-1].FilePath + } + if len(gone) == 0 { + return + } + if err := i.cfg.State.Delete(ctx, gone); err != nil { + slog.WarnContext(ctx, "could not drop peer import rows whose files are gone", "error", err) + } +} diff --git a/internal/peernet/bundleimport/release.go b/internal/peernet/bundleimport/release.go new file mode 100644 index 000000000..6d6039897 --- /dev/null +++ b/internal/peernet/bundleimport/release.go @@ -0,0 +1,90 @@ +package bundleimport + +import ( + "context" + "errors" + "fmt" + "path" + "strings" + "time" + + "github.com/kipsilabs/altmount/internal/importer/filesystem" + "github.com/kipsilabs/altmount/internal/peernet/bundle" +) + +// directImportTimeout bounds one shared direct import (fetch plus accept). +const directImportTimeout = 3 * time.Minute + +// ImportRelease is the direct import without an NZB: fetch the bundle of +// releaseID, accept it, and record it as a normal import (queue and history +// entry, post-processing, health scheduling). The folder is the target +// category plus the bundle's sanitised title, which is also the title shown. +// It is idempotent per release and concurrent requests for one release +// coalesce into one fetch and one import. +func (i *Importer) ImportRelease(ctx context.Context, releaseID string, target Target) (*Result, error) { + if i.cfg.Fetcher == nil { + return i.rejected(ReasonNoNetwork, errors.New("no fetcher installed")) + } + v, err, _ := i.flight.Do(releaseID, func() (any, error) { + // The work is shared by every caller of this release, so it must not + // die with the first caller's context: detach, with a bounded life. + work, cancel := context.WithTimeout(context.WithoutCancel(ctx), directImportTimeout) + defer cancel() + return i.importRelease(work, releaseID, target) + }) + if err != nil { + return nil, err + } + return v.(*Result), nil +} + +func (i *Importer) importRelease(ctx context.Context, releaseID string, target Target) (*Result, error) { + b, origin, err := i.cfg.Fetcher.FetchBundle(ctx, releaseID) + if err != nil || b == nil { + return i.rejected(ReasonUnavailable, err) + } + if b.ReleaseID.String() != releaseID { + return i.rejected(ReasonInvalid, errors.New("bundle is for another release")) + } + title := bundle.SanitizeFilename(b.Title) + name, ok := filesystem.SafeFolderName(title) + if !ok { + return i.rejected(ReasonBadTarget, errors.New("the bundle title cannot name a folder")) + } + if target.Dir == "" && target.DirFunc != nil { + target.Dir = target.DirFunc(title) + if target.Dir == "" { + return i.rejected(ReasonBadTarget, errors.New("the caller refused the bundle title")) + } + } + if target.Dir == "" { + category := strings.Trim(strings.ReplaceAll(target.Category, `\`, "/"), "/") + target.Dir = path.Join(category, name) + } + if d, err := cleanTarget(target.Dir); err != nil || path.Base(d) == "." || path.Base(d) == ".." { + return i.rejected(ReasonBadTarget, errors.New("the target folder is not a plain folder below its category")) + } + res, err := i.Accept(ctx, Request{Bundle: b, Origin: origin, Target: target}) + if err != nil { + return nil, err + } + if res.AlreadyImported || i.cfg.Recorder == nil { + return res, nil + } + queueID, err := i.cfg.Recorder.RecordPeerImport(ctx, ImportRecord{ + Title: title, Category: target.Category, TargetPath: target.TargetPath, ResultingPath: res.Dir, + WrittenPaths: res.Paths, TotalSize: res.TotalSize, ReleaseID: releaseID, SkipArr: target.SkipArr, Metadata: target.Metadata, DownloadID: target.DownloadID, + }) + if err != nil { + bg := context.WithoutCancel(ctx) + for _, p := range res.Paths { + _ = i.cfg.Library.DeleteFileMetadata(bg, p) + } + _ = i.cfg.State.Delete(bg, res.Paths) + return nil, fmt.Errorf("bundleimport: record import: %w", err) + } + if err := i.AttachQueue(ctx, res.Paths, queueID); err != nil { + return nil, fmt.Errorf("bundleimport: link queue item: %w", err) + } + return res, nil +} diff --git a/internal/peernet/bundleimport/sample.go b/internal/peernet/bundleimport/sample.go new file mode 100644 index 000000000..1dacefa5a --- /dev/null +++ b/internal/peernet/bundleimport/sample.go @@ -0,0 +1,210 @@ +package bundleimport + +import ( + "context" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "io" + "sync" + + "github.com/javi11/nntppool/v5" + + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/pool" +) + +// YencHeader is what an article's yEnc headers say about its place in a file: +// the decoded size of the part, its 0-based offset in the file and the size of +// the whole file. +type YencHeader struct { + PartBegin int64 + PartSize int64 + FileSize int64 +} + +// SegmentHeaders reads the yEnc headers of one article from the user's own +// providers. +type SegmentHeaders interface { + SegmentHeader(ctx context.Context, messageID string) (YencHeader, error) +} + +type poolHeaders struct{ p pool.Manager } + +// NewPoolHeaderReader reads headers through the connection pool. The pool +// cannot stop after the headers, so a read costs the whole article (about +// 0.75 MB); the sample count is therefore bounded by Policy.YencSamples. +func NewPoolHeaderReader(p pool.Manager) SegmentHeaders { return poolHeaders{p: p} } + +func (h poolHeaders) SegmentHeader(ctx context.Context, id string) (YencHeader, error) { + cli, err := h.p.GetPool() + if err != nil { + return YencHeader{}, err + } + body, err := cli.Fetch(ctx, nntppool.Req{MessageID: id, Writer: io.Discard}) + if err != nil { + return YencHeader{}, err + } + return YencHeader{PartBegin: body.YEnc.PartBegin, PartSize: body.YEnc.PartSize, FileSize: body.YEnc.FileSize}, nil +} + +// sampleTable is one segment table read as a whole file or archive volume. +type sampleTable struct { + name string + ids []string + sizes []int64 + prefix []int64 // prefix[i] = sum of sizes[:i] + total int64 +} + +type sample struct{ table, pos int } + +// tablesOf lists the file and outer-volume tables of a bundle, each outer +// volume once even when several files share it. +func tablesOf(b *bundle.Bundle) []sampleTable { + var out []sampleTable + seen := map[string]struct{}{} + add := func(name string, segs []bundle.Segment) { + if len(segs) == 0 { + return + } + first := b.Pool[segs[0].Ref] + if _, dup := seen[first]; dup { + return + } + seen[first] = struct{}{} + t := sampleTable{name: name, ids: make([]string, len(segs)), sizes: make([]int64, len(segs)), prefix: make([]int64, len(segs))} + for i, s := range segs { + t.ids[i], t.sizes[i], t.prefix[i] = b.Pool[s.Ref], s.Size, t.total + t.total += s.Size + } + out = append(out, t) + } + for i := range b.Files { + f := &b.Files[i] + add(f.Name, f.Segments) + for j := range f.Outer { + add(fmt.Sprintf("%s (volume %d)", f.Name, j+1), f.Outer[j].Segments) + } + } + return out +} + +// randN returns a uniform value in [0, n) from the operating system's random +// source, so a publisher cannot predict which articles will be read. +func randN(n int) int { + if n <= 1 { + return 0 + } + var buf [8]byte + for { + if _, err := rand.Read(buf[:]); err != nil { + panic("bundleimport: no system randomness: " + err.Error()) + } + v := binary.BigEndian.Uint64(buf[:]) + // Rejection sampling keeps the result unbiased. + if limit := ^uint64(0) - (^uint64(0))%uint64(n); v < limit { + return int(v % uint64(n)) + } + } +} + +func shuffle[T any](s []T) { + for i := len(s) - 1; i > 0; i-- { + j := randN(i + 1) + s[i], s[j] = s[j], s[i] + } +} + +// planSamples picks, per table, the first and last segment and perRandom +// random ones in between, within a total cap. When the cap is smaller than the +// number of tables, a random subset of tables is read. +func planSamples(tables []sampleTable, perRandom, limit int) []sample { + order := make([]int, len(tables)) + for i := range order { + order[i] = i + } + shuffle(order) + var plan []sample + taken := map[sample]struct{}{} + push := func(t, pos int) { + s := sample{t, pos} + if _, dup := taken[s]; dup || len(plan) >= limit { + return + } + taken[s] = struct{}{} + plan = append(plan, s) + } + for _, t := range order { + n := len(tables[t].ids) + push(t, 0) + if n > 1 { + push(t, n-1) + } + } + for _, t := range order { + n := len(tables[t].ids) + for k, tries := 0, 0; k < perRandom && n > 2 && tries < 16*perRandom; tries++ { + before := len(plan) + push(t, 1+randN(n-2)) + if len(plan) > before { + k++ + } + } + } + return plan +} + +// verifyYenc reads the yEnc headers of sampled articles and requires each to +// agree with the layout: decoded part size equals the declared size, the part +// begins at the cumulative offset the layout implies, and the file total equals +// the sum of the table's sizes. A mismatch is the publisher's doing. +func (i *Importer) verifyYenc(ctx context.Context, b *bundle.Bundle) (Reason, error) { + if i.cfg.Headers == nil { + return ReasonProbeInconclusive, errors.New("no way to read article headers") + } + tables := tablesOf(b) + plan := planSamples(tables, i.cfg.Policy.YencRandom(), i.cfg.Policy.YencSamples()) + type verdict struct { + reason Reason + err error + } + results := make([]verdict, len(plan)) + var wg sync.WaitGroup + sem := make(chan struct{}, 4) + for k, s := range plan { + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + t := tables[s.table] + h, err := i.cfg.Headers.SegmentHeader(ctx, t.ids[s.pos]) + switch { + case errors.Is(err, nntppool.ErrArticleNotFound): + results[k] = verdict{ReasonProbeMissing, fmt.Errorf("%s: article %d is missing", t.name, s.pos)} + case err != nil: + results[k] = verdict{ReasonProbeInconclusive, fmt.Errorf("%s: %w", t.name, err)} + case h.PartSize != t.sizes[s.pos] || h.PartBegin != t.prefix[s.pos] || h.FileSize != t.total: + results[k] = verdict{ReasonYencMismatch, fmt.Errorf("%s: article %d says part %d at %d of %d, layout says %d at %d of %d", + t.name, s.pos, h.PartSize, h.PartBegin, h.FileSize, t.sizes[s.pos], t.prefix[s.pos], t.total)} + } + }() + } + wg.Wait() + var first *verdict + for k := range results { + r := &results[k] + if r.reason == ReasonYencMismatch { + return r.reason, r.err // a contradiction outranks a missing article + } + if r.reason != "" && first == nil { + first = r + } + } + if first != nil { + return first.reason, first.err + } + return "", nil +} diff --git a/internal/peernet/bundleimport/sizes.go b/internal/peernet/bundleimport/sizes.go new file mode 100644 index 000000000..84de194be --- /dev/null +++ b/internal/peernet/bundleimport/sizes.go @@ -0,0 +1,77 @@ +package bundleimport + +import ( + "errors" + "fmt" + + "github.com/kipsilabs/altmount/internal/peernet/bundle" +) + +// Encoded versus decoded sizes. An NZB's is the yEnc-ENCODED +// size: escaping and line breaks add roughly 2 to 3 percent, plus a header of a +// couple of hundred bytes per article, so a decoded size is at most the encoded one and +// not much below it. The band is deliberately loose (the importer's own +// estimate uses a 1 percent tolerance on top of this); it catches layouts that +// claim sizes the user's own NZB contradicts, not small lies. +const ( + minDecodedPermille = 900 // decoded >= 0.90 * encoded - sizeSlack + sizeSlack = 512 // header and short-segment slack, per table +) + +func plausibleDecoded(decoded, encoded, parts int64) bool { + return decoded <= encoded && decoded*1000 >= encoded*minDecodedPermille-sizeSlack*parts*1000 +} + +// checkSizes compares every segment table's declared decoded sizes with the +// encoded sizes of the user's NZB (nzb scheme only) and each table's sum +// likewise. +func checkSizes(b *bundle.Bundle, req Request) error { + if req.ExpectedNZBSizes == nil || b.Scheme != bundle.SchemeNZB { + return nil + } + if len(req.ExpectedNZBSizes) != len(req.ExpectedNZB) { + return errors.New("NZB sizes do not match the NZB list") + } + enc := map[string]int64{} + for i, ids := range req.ExpectedNZB { + if len(ids) != len(req.ExpectedNZBSizes[i]) { + return errors.New("NZB sizes do not match the NZB list") + } + for k, id := range ids { + enc[id] = req.ExpectedNZBSizes[i][k] + } + } + table := func(name string, segs []bundle.Segment) error { + var dsum, esum int64 + for _, s := range segs { + e, ok := enc[b.Pool[s.Ref]] + if !ok { + return fmt.Errorf("%s: a segment is not in the NZB", name) + } + if e <= 0 { + return nil // an NZB without segment sizes cannot bound anything + } + if !plausibleDecoded(s.Size, e, 1) { + return fmt.Errorf("%s: declared size %d against %d encoded bytes in your NZB", name, s.Size, e) + } + dsum += s.Size + esum += e + } + if !plausibleDecoded(dsum, esum, int64(len(segs))) { + return fmt.Errorf("%s: declared %d bytes against %d encoded bytes in your NZB", name, dsum, esum) + } + return nil + } + for i := range b.Files { + f := &b.Files[i] + if err := table(f.Name, f.Segments); err != nil { + return err + } + for j := range f.Outer { + if err := table(fmt.Sprintf("%s outer %d", f.Name, j), f.Outer[j].Segments); err != nil { + return err + } + } + } + return nil +} diff --git a/internal/peernet/bundleimport/sizes_test.go b/internal/peernet/bundleimport/sizes_test.go new file mode 100644 index 000000000..5dc1bba18 --- /dev/null +++ b/internal/peernet/bundleimport/sizes_test.go @@ -0,0 +1,59 @@ +package bundleimport + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/kipsilabs/altmount/internal/peernet/bundle" +) + +// nzbFor returns the NZB view of b: the ids and, per segment, an encoded size +// derived from its decoded size with encode. +func nzbFor(t *testing.T, e *env, b *bundle.Bundle, encode func(int64) int64) ([][]string, [][]int64) { + t.Helper() + loc, err := bundle.ToLocal(b, e.imp.cfg.Now()) + require.NoError(t, err) + dec := map[string]int64{} + for _, f := range b.Files { + for _, s := range f.Segments { + dec[b.Pool[s.Ref]] = s.Size + } + } + sizes := make([][]int64, len(loc.NZBFiles)) + for i, ids := range loc.NZBFiles { + for _, id := range ids { + sizes[i] = append(sizes[i], encode(dec[id])) + } + } + return loc.NZBFiles, sizes +} + +func TestAccept_SizesMustBePlausibleAgainstTheUsersNZB(t *testing.T) { + cases := map[string]struct { + encode func(int64) int64 + ok bool + }{ + "yenc sized": {func(n int64) int64 { return n + n/32 + 20 }, true}, + "declared far too big": {func(n int64) int64 { return n / 4 }, false}, + "declared far too tiny": {func(n int64) int64 { return n * 1000 }, false}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("sz", mkvSpec("movie.mkv")) + req := e.req(b) + req.ExpectedNZB, req.ExpectedNZBSizes = nzbFor(t, e, b, tc.encode) + _, err := e.imp.Accept(context.Background(), req) + if tc.ok { + require.NoError(t, err) + return + } + require.Error(t, err) + assert.Equal(t, ReasonSizeImplausible, ReasonOf(err)) + e.assertNothingBehind() + }) + } +} diff --git a/internal/peernet/bundleimport/staging.go b/internal/peernet/bundleimport/staging.go new file mode 100644 index 000000000..4979a7c87 --- /dev/null +++ b/internal/peernet/bundleimport/staging.go @@ -0,0 +1,85 @@ +package bundleimport + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/kipsilabs/altmount/internal/config" + "github.com/kipsilabs/altmount/internal/contentverify" + "github.com/kipsilabs/altmount/internal/metadata" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/nzbfilesystem" + "github.com/kipsilabs/altmount/internal/nzbfilesystem/segcache" + "github.com/kipsilabs/altmount/internal/pool" +) + +// FSStagerDeps are what the real stager reads through. +type FSStagerDeps struct { + // Pool and Config are the same the serving stack uses. + Pool pool.Manager + Config config.ConfigGetter + // Cache is the shared segment cache (may be nil). + Cache *segcache.Source + // Root is the directory staging areas are created in. It must be outside + // the metadata root, so nothing staged can ever be listed or mounted. + Root string +} + +// FSStager stages files in a private metadata service rooted in a temporary +// directory and opens them through a real NzbFilesystem built over it, which +// is the same reader stack that will later serve the files. The staging +// metadata service has no file observer (so nothing is indexed), no health +// repository (so a read problem cannot write health state) and no ARR or +// rclone client. +type FSStager struct{ deps FSStagerDeps } + +// NewFSStager creates the real stager. +func NewFSStager(deps FSStagerDeps) *FSStager { return &FSStager{deps: deps} } + +type noopTracker struct{} + +func (noopTracker) Add(_, _, _, _, _ string, _ int64) string { return "" } +func (noopTracker) UpdateProgress(string, int64) {} +func (noopTracker) UpdateDownloadProgress(string, int64) {} +func (noopTracker) UpdateCurrentOffset(string, int64) {} +func (noopTracker) UpdateBufferedOffset(string, int64) {} +func (noopTracker) Remove(string) {} +func (noopTracker) IncArticlesDownloaded() {} +func (noopTracker) IncArticlesPosted() {} + +type fsStaging struct { + dir string + ms *metadata.MetadataService + fs *nzbfilesystem.NzbFilesystem +} + +// NewStaging creates a fresh, empty staging area. +func (s *FSStager) NewStaging() (Staging, error) { + if s.deps.Pool == nil || s.deps.Config == nil { + return nil, errors.New("bundleimport: staging needs a pool and a configuration") + } + root := s.deps.Root + if root == "" { + root = filepath.Join(os.TempDir(), "altmount-peernet-staging") + } + if err := os.MkdirAll(root, 0o700); err != nil { + return nil, fmt.Errorf("bundleimport: create staging root: %w", err) + } + dir, err := os.MkdirTemp(root, "stage-") + if err != nil { + return nil, fmt.Errorf("bundleimport: create staging area: %w", err) + } + ms := metadata.NewMetadataService(dir) + remote := nzbfilesystem.NewMetadataRemoteFile(ms, nil, nil, nil, s.deps.Pool, s.deps.Config, noopTracker{}, s.deps.Cache) + return &fsStaging{dir: dir, ms: ms, fs: nzbfilesystem.NewNzbFilesystem(remote)}, nil +} + +func (s *fsStaging) WriteFileMetadata(virtualPath string, m *metapb.FileMetadata) error { + return s.ms.WriteFileMetadata(virtualPath, m) +} + +func (s *fsStaging) Opener() contentverify.Opener { return s.fs } + +func (s *fsStaging) Close() error { return os.RemoveAll(s.dir) } diff --git a/internal/peernet/bundleimport/structure.go b/internal/peernet/bundleimport/structure.go new file mode 100644 index 000000000..55db1ba08 --- /dev/null +++ b/internal/peernet/bundleimport/structure.go @@ -0,0 +1,502 @@ +package bundleimport + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "time" + + "github.com/javi11/nntppool/v5" + + "github.com/kipsilabs/altmount/internal/contentverify" + "github.com/kipsilabs/altmount/internal/utils" +) + +// Structural media verification. The head signature check (contentverify) +// only proves the first bytes decode. This reads the container's own index at +// several positions, including the end of the file, through the staged layout, +// with a small bounded number of small reads: +// +// Matroska / WebM EBML header, Segment size against the file size, and the +// SeekHead: every element it points at (Info, Tracks, Cues, +// ...) must start with the ID the SeekHead promises, which +// reaches the Cues at the tail. A file muxed for streaming +// has no SeekHead (and may have an unknown-size Segment): it +// needs a level-1 element first, a Segment that ends at the +// end of the file when its size is known, and readable +// quartiles and tail. +// MP4 / MOV the whole top-level box chain must land exactly on the +// end of the file (a zero-filled tail of up to 64 KiB, which +// some writers pad with, is allowed) and contain moov and +// mdat. A fragmented file (moof boxes) has thousands of +// boxes: its first boxes are walked, and a box of the +// fragment family must end exactly at the end of the file. +// MPEG-TS / M2TS the sync byte on the packet grid (188, 192 with the +// 4-byte timestamp, or 204 bytes) at the start, quartiles and +// the last packet. +// anything else full-length reads at the quartiles and the last bytes. +// +// There is no deeper media parser in the repository (no MediaStructure probe), +// so this is the strongest check available without adding one. + +const ( + structReadTimeout = 20 * time.Second + maxBoxes = 4096 + maxSeekEntries = 16 + tailWindow = 4096 +) + +var errStructure = errors.New("media structure is not valid") + +type readerAt struct { + ctx context.Context + f io.ReaderAt + size int64 + n int // reads made, bounded +} + +const maxStructReads = 64 + +func (r *readerAt) read(off int64, n int) ([]byte, error) { + if r.n++; r.n > maxStructReads { + return nil, fmt.Errorf("%w: too many reads", errStructure) + } + if off < 0 || off >= r.size { + return nil, fmt.Errorf("%w: offset %d outside the file", errStructure, off) + } + n = int(min(int64(n), r.size-off)) + buf := make([]byte, n) + got, err := r.f.ReadAt(buf, off) + if got == n { + return buf, nil + } + if err == nil || errors.Is(err, io.EOF) { + return nil, fmt.Errorf("%w: short read at %d", errStructure, off) + } + return nil, err +} + +// verifyStructure runs the container check for the staged file name. +func verifyStructure(ctx context.Context, op contentverify.Opener, name string, size int64) error { + ctx, cancel := context.WithTimeout(ctx, structReadTimeout) + defer cancel() + ctx = context.WithValue(ctx, utils.MaxPrefetchKey, 1) + ctx = context.WithValue(ctx, utils.SuppressStreamTrackingKey, true) + f, err := op.OpenFile(ctx, name, os.O_RDONLY, 0) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + r := &readerAt{ctx: ctx, f: f, size: size} + head, err := r.read(0, 16) + if err != nil { + return err + } + switch { + case bytes.HasPrefix(head, []byte{0x1A, 0x45, 0xDF, 0xA3}): + return verifyMatroska(r) + case len(head) >= 8 && string(head[4:8]) == "ftyp": + return verifyMP4(r) + case tsStride(r) != 0: + return verifyTS(r, tsStride(r)) + } + return verifyWindows(r) +} + +// ---- generic ---- + +func verifyWindows(r *readerAt) error { + for _, q := range []int64{0, 1, 2, 3} { + off := r.size / 4 * q + if _, err := r.read(off, 512); err != nil { + return err + } + } + _, err := r.read(max(r.size-tailWindow, 0), tailWindow) + return err +} + +// ---- MP4 ---- + +// maxFragHeadBoxes is how many top-level boxes of a fragmented file are walked +// from the start before the end is checked on its own. +const maxFragHeadBoxes = 8 + +// fragTailWindows are the sizes of the tail windows searched for the last box of +// a fragmented file; the second covers fragments of several megabytes. +var fragTailWindows = []int64{4 << 20, 16 << 20} + +// maxZeroTail is the zero padding some writers leave after the last box. +const maxZeroTail = 64 << 10 + +// fragmentBoxTypes are the box types a fragmented file can end with. +var fragmentBoxTypes = map[string]bool{ + "moof": true, "mdat": true, "mfra": true, "free": true, "skip": true, + "styp": true, "sidx": true, "emsg": true, "prft": true, +} + +func verifyMP4(r *readerAt) error { + var off int64 + seen := map[string]bool{} + walked := 0 + for boxes := 0; off < r.size; boxes++ { + if boxes > maxBoxes { + return fmt.Errorf("%w: too many boxes", errStructure) + } + hdr, err := r.read(off, 16) + if err != nil { + return err + } + if len(hdr) < 8 { + return fmt.Errorf("%w: truncated box header", errStructure) + } + if zeroTail, err := r.isZeroTail(off, hdr); err != nil { + return err + } else if zeroTail { + break + } + size := int64(uint32(hdr[0])<<24 | uint32(hdr[1])<<16 | uint32(hdr[2])<<8 | uint32(hdr[3])) + typ := string(hdr[4:8]) + if !printableBoxType(typ) { + return fmt.Errorf("%w: bad box type at %d", errStructure, off) + } + switch size { + case 0: + size = r.size - off // runs to the end of the file + case 1: + if len(hdr) < 16 { + return fmt.Errorf("%w: truncated large box", errStructure) + } + size = 0 + for _, b := range hdr[8:16] { + size = size<<8 | int64(b) + } + } + if size < 8 || off+size > r.size || off+size < off { + return fmt.Errorf("%w: box %q at %d runs past the file", errStructure, typ, off) + } + seen[typ] = true + off += size + walked++ + // A fragmented file: thousands of moof/mdat pairs cannot all be walked + // with a bounded number of reads. Walk the head, then check the end. + if seen["moof"] && walked >= maxFragHeadBoxes && off < r.size { + if !seen["moov"] || !seen["mdat"] { + return fmt.Errorf("%w: fragmented file without moov and mdat", errStructure) + } + return verifyFragmentedTail(r) + } + } + if !seen["moov"] || !seen["mdat"] || (off != r.size && !r.tailIsZero(off)) { + return fmt.Errorf("%w: box chain incomplete", errStructure) + } + return nil +} + +// isZeroTail reports whether everything from off to the end of the file is +// zero and short (padding after the last box). hdr is the 16 bytes at off. +func (r *readerAt) isZeroTail(off int64, hdr []byte) (bool, error) { + for _, c := range hdr { + if c != 0 { + return false, nil + } + } + return r.tailIsZero(off), nil +} + +func (r *readerAt) tailIsZero(off int64) bool { + if r.size-off > maxZeroTail || off >= r.size { + return off >= r.size + } + buf, err := r.read(off, int(r.size-off)) + if err != nil { + return false + } + for _, c := range buf { + if c != 0 { + return false + } + } + return true +} + +// verifyFragmentedTail requires a fragment-family box that ends exactly at the +// end of the file, found in a tail window. Random bytes at the end do not +// satisfy it: a header with a known type whose size lands on the last byte. +func verifyFragmentedTail(r *readerAt) error { + for _, w := range fragTailWindows { + n := min(w, r.size) + base := r.size - n + buf, err := r.read(base, int(n)) + if err != nil { + return err + } + for o := 0; o+8 <= len(buf); o++ { + size := int64(uint32(buf[o])<<24 | uint32(buf[o+1])<<16 | uint32(buf[o+2])<<8 | uint32(buf[o+3])) + if size < 8 || int64(o)+size != int64(len(buf)) { + continue + } + if fragmentBoxTypes[string(buf[o+4:o+8])] { + return nil + } + } + if n == r.size { + break + } + } + return fmt.Errorf("%w: no box ends at the end of the fragmented file", errStructure) +} + +func printableBoxType(t string) bool { + for _, c := range []byte(t) { + letter := c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' + if !letter && c != ' ' && c != '_' && c != 0xA9 { + return false + } + } + return true +} + +// ---- MPEG-TS ---- + +func tsStride(r *readerAt) int64 { + buf, err := r.read(0, 4*204) + if err != nil { + return 0 + } + for _, c := range []struct{ stride, sync int }{{188, 0}, {192, 4}, {204, 0}} { + ok := len(buf) > c.sync+3*c.stride + for k := 0; ok && k < 4; k++ { + ok = buf[c.sync+k*c.stride] == 0x47 + } + if ok { + return int64(c.stride) + } + } + return 0 +} + +func verifyTS(r *readerAt, stride int64) error { + sync := int64(0) + if stride == 192 { + sync = 4 + } + packets := r.size / stride + if packets < 8 { + return fmt.Errorf("%w: too short for a transport stream", errStructure) + } + for _, q := range []int64{1, 2, 3} { + if err := tsSync(r, packets/4*q*stride+sync); err != nil { + return err + } + } + return tsSync(r, (packets-1)*stride+sync) +} + +func tsSync(r *readerAt, off int64) error { + b, err := r.read(off, 1) + if err != nil { + return err + } + if b[0] != 0x47 { + return fmt.Errorf("%w: lost transport stream sync at %d", errStructure, off) + } + return nil +} + +// ---- Matroska / WebM ---- + +// ebmlVint parses an EBML variable-length integer. keepMarker keeps the length +// marker bit (element IDs); unknown reports the all-ones "unknown size" value. +func ebmlVint(b []byte, keepMarker bool) (v uint64, n int, unknown bool, ok bool) { + if len(b) == 0 || b[0] == 0 { + return 0, 0, false, false + } + n = 1 + for mask := byte(0x80); b[0]&mask == 0; mask >>= 1 { + n++ + } + if n > 8 || len(b) < n { + return 0, 0, false, false + } + v = uint64(b[0]) + if !keepMarker { + v &= (1 << (8 - uint(n))) - 1 + } + allOnes := (b[0] | byte(0xFF<<(8-uint(n)))) == 0xFF + for _, c := range b[1:n] { + v = v<<8 | uint64(c) + allOnes = allOnes && c == 0xFF + } + return v, n, !keepMarker && allOnes, true +} + +const ( + idEBML = 0x1A45DFA3 + idSegment = 0x18538067 + idSeekHead = 0x114D9B74 + idSeek = 0x4DBB + idSeekID = 0x53AB + idSeekPos = 0x53AC + idVoid = 0xEC +) + +func verifyMatroska(r *readerAt) error { + hdr, err := r.read(0, 64) + if err != nil { + return err + } + _, idLen, _, ok := ebmlVint(hdr, true) + if !ok { + return fmt.Errorf("%w: bad EBML header", errStructure) + } + hsize, sLen, _, ok := ebmlVint(hdr[idLen:], false) + if !ok || hsize == 0 || hsize > 1024 { + return fmt.Errorf("%w: bad EBML header size", errStructure) + } + segOff := int64(idLen+sLen) + int64(hsize) + seg, err := r.read(segOff, 16) + if err != nil { + return err + } + id, idn, _, ok := ebmlVint(seg, true) + if !ok || id != idSegment { + return fmt.Errorf("%w: no Segment after the EBML header", errStructure) + } + ssize, sn, unknown, ok := ebmlVint(seg[idn:], false) + if !ok { + return fmt.Errorf("%w: bad Segment size", errStructure) + } + start := segOff + int64(idn+sn) + if !unknown { + end := start + int64(ssize) + if end < start || end > r.size || r.size-end > 1<<16 { + return fmt.Errorf("%w: Segment ends at %d, file is %d bytes", errStructure, end, r.size) + } + } + return verifySeekHead(r, start) +} + +// level1IDs are the elements that can start a Segment's children. +var level1IDs = map[uint64]bool{ + idSeekHead: true, 0x1549A966: true /* Info */, 0x1654AE6B: true /* Tracks */, 0x1F43B675: true, /* Cluster */ + 0x1C53BB6B: true /* Cues */, 0x1254C367: true /* Tags */, 0x1043A770: true /* Chapters */, 0x1941A469: true, /* Attachments */ + idVoid: true, 0xBF: true, /* CRC-32 */ +} + +func verifySeekHead(r *readerAt, segStart int64) error { + buf, err := r.read(segStart, 1024) + if err != nil { + return err + } + // The SeekHead is the first meaningful child (a Void may precede it). A + // file muxed for streaming has none. + for len(buf) > 0 { + id, idn, _, ok := ebmlVint(buf, true) + if !ok || !level1IDs[id] { + return fmt.Errorf("%w: unreadable element after the Segment header", errStructure) + } + sz, sn, _, ok := ebmlVint(buf[idn:], false) + if !ok || uint64(len(buf)-idn-sn) < sz && id == idSeekHead { + return fmt.Errorf("%w: bad SeekHead", errStructure) + } + body := idn + sn + switch id { + case idVoid: + if uint64(len(buf)-body) < sz { + return verifyNoSeekHead(r) + } + buf = buf[body+int(sz):] + case idSeekHead: + return checkSeeks(r, segStart, buf[body:body+int(sz)]) + default: + return verifyNoSeekHead(r) + } + } + return verifyNoSeekHead(r) +} + +// verifyNoSeekHead checks a file that has no SeekHead (a live or streaming +// mux): a level-1 element opens the Segment (checked by the caller), the +// Segment size already had to land on the end of the file when it is known, and +// the quartiles and the tail must be readable through the layout. +func verifyNoSeekHead(r *readerAt) error { return verifyWindows(r) } + +func checkSeeks(r *readerAt, segStart int64, head []byte) error { + entries := 0 + for len(head) > 0 && entries < maxSeekEntries { + id, idn, _, ok := ebmlVint(head, true) + sz, sn, _, ok2 := ebmlVint(head[min(idn, len(head)):], false) + if !ok || !ok2 || id != idSeek || uint64(len(head)-idn-sn) < sz { + return fmt.Errorf("%w: bad Seek entry", errStructure) + } + entry := head[idn+sn : idn+sn+int(sz)] + head = head[idn+sn+int(sz):] + wantID, pos, err := parseSeek(entry) + if err != nil { + return err + } + got, err := r.read(segStart+pos, len(wantID)) + if err != nil { + return err + } + if !bytes.Equal(got, wantID) { + return fmt.Errorf("%w: the SeekHead promises %x at %d, the file has %x", errStructure, wantID, segStart+pos, got) + } + entries++ + } + if entries == 0 { + return fmt.Errorf("%w: empty SeekHead", errStructure) + } + return nil +} + +func parseSeek(b []byte) (id []byte, pos int64, err error) { + havePos := false + for len(b) > 0 { + eid, idn, _, ok := ebmlVint(b, true) + sz, sn, _, ok2 := ebmlVint(b[min(idn, len(b)):], false) + if !ok || !ok2 || uint64(len(b)-idn-sn) < sz { + return nil, 0, fmt.Errorf("%w: bad Seek entry", errStructure) + } + body := b[idn+sn : idn+sn+int(sz)] + b = b[idn+sn+int(sz):] + switch eid { + case idSeekID: + if len(body) < 1 || len(body) > 4 { + return nil, 0, fmt.Errorf("%w: bad SeekID", errStructure) + } + id = append([]byte(nil), body...) + case idSeekPos: + if len(body) < 1 || len(body) > 8 { + return nil, 0, fmt.Errorf("%w: bad SeekPosition", errStructure) + } + var v uint64 + for _, c := range body { + v = v<<8 | uint64(c) + } + if v > 1<<50 { + return nil, 0, fmt.Errorf("%w: SeekPosition out of range", errStructure) + } + pos, havePos = int64(v), true + } + } + if id == nil || !havePos { + return nil, 0, fmt.Errorf("%w: incomplete Seek entry", errStructure) + } + return id, pos, nil +} + +// structureReason maps a structure error to the rejection reason. +func structureReason(err error) Reason { + switch { + case errors.Is(err, errStructure): + return ReasonStructureInvalid + case errors.Is(err, nntppool.ErrArticleNotFound): + return ReasonMediaMissing + } + return ReasonMediaInconclusive +} diff --git a/internal/peernet/bundleimport/structure_test.go b/internal/peernet/bundleimport/structure_test.go new file mode 100644 index 000000000..81efbd65f --- /dev/null +++ b/internal/peernet/bundleimport/structure_test.go @@ -0,0 +1,80 @@ +package bundleimport + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func acceptPlain(t *testing.T, plain []byte, name string) error { + t.Helper() + e := newEnv(t) + b := e.mkBundle("st", fileSpec{name: name, plain: plain, segSize: 128}) + _, err := e.imp.Accept(context.Background(), e.req(b)) + if err != nil { + e.assertNothingBehind() + } + return err +} + +func TestStructure_MatroskaIndexAtTheTailMustHold(t *testing.T) { + good := mkvPlain(1024) + require.NoError(t, acceptPlain(t, good, "a.mkv")) + + t.Run("SeekHead points at the wrong element", func(t *testing.T) { + bad := mkvPlain(1024) + // Break the Cues element at the tail (its ID), keep the head intact. + bad[len(bad)-9] ^= 0xFF + err := acceptPlain(t, bad, "a.mkv") + require.Error(t, err) + assert.Equal(t, ReasonStructureInvalid, ReasonOf(err)) + }) + t.Run("Segment claims more than the file holds", func(t *testing.T) { + bad := mkvPlain(1024) + bad[16+7] += 9 // low byte of the Segment size + err := acceptPlain(t, bad, "a.mkv") + require.Error(t, err) + assert.Equal(t, ReasonStructureInvalid, ReasonOf(err)) + }) +} + +func mp4Box(typ string, payload int) []byte { + n := 8 + payload + b := []byte{byte(n >> 24), byte(n >> 16), byte(n >> 8), byte(n)} + b = append(b, typ...) + return append(b, make([]byte, payload)...) +} + +func TestStructure_MP4BoxChainMustEndAtTheEndOfTheFile(t *testing.T) { + var good []byte + good = append(good, mp4Box("ftyp", 16)...) + good = append(good, mp4Box("moov", 200)...) + good = append(good, mp4Box("mdat", 600)...) + copy(good[8:], "isom") + require.NoError(t, acceptPlain(t, good, "a.mp4")) + + bad := append(append([]byte{}, good...), 1, 2, 3) // trailing bytes beyond the last box + err := acceptPlain(t, bad, "a.mp4") + require.Error(t, err) + assert.Equal(t, ReasonStructureInvalid, ReasonOf(err)) + + noMoov := append(append([]byte{}, mp4Box("ftyp", 16)...), mp4Box("mdat", 800)...) + err = acceptPlain(t, noMoov, "a.mp4") + require.Error(t, err) + assert.Equal(t, ReasonStructureInvalid, ReasonOf(err)) +} + +func TestStructure_TransportStreamNeedsSyncAtTheEnd(t *testing.T) { + ts := make([]byte, 188*40) + for k := 0; k < 40; k++ { + ts[k*188] = 0x47 + } + require.NoError(t, acceptPlain(t, ts, "a.ts")) + bad := append([]byte{}, ts...) + bad[188*39] = 0 + err := acceptPlain(t, bad, "a.ts") + require.Error(t, err) + assert.Equal(t, ReasonStructureInvalid, ReasonOf(err)) +} diff --git a/internal/peernet/bundleimport/structure_variants_test.go b/internal/peernet/bundleimport/structure_variants_test.go new file mode 100644 index 000000000..e63a6e370 --- /dev/null +++ b/internal/peernet/bundleimport/structure_variants_test.go @@ -0,0 +1,231 @@ +package bundleimport + +import ( + "bytes" + "context" + "os" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// memOpener serves one in-memory file through the Opener the check reads from. +type memOpener struct{ fs afero.Fs } + +func (m memOpener) OpenFile(_ context.Context, name string, flag int, perm os.FileMode) (afero.File, error) { + return m.fs.OpenFile(name, flag, perm) +} + +func structureOf(t *testing.T, data []byte, name string) error { + t.Helper() + fs := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, name, data, 0o644)) + return verifyStructure(context.Background(), memOpener{fs}, name, int64(len(data))) +} + +func requireAccepted(t *testing.T, data []byte, name string) { + t.Helper() + assert.NoError(t, structureOf(t, data, name)) +} + +func requireRejected(t *testing.T, data []byte, name string) { + t.Helper() + err := structureOf(t, data, name) + require.Error(t, err) + assert.ErrorIs(t, err, errStructure) +} + +// ---- Matroska ---- + +func ebmlSize(n int) []byte { // 8-byte EBML size + return []byte{0x01, 0, byte(n >> 40), byte(n >> 32), byte(n >> 24), byte(n >> 16), byte(n >> 8), byte(n)} +} + +// mkvNoSeekHead is a streaming-style file: EBML header, a Segment (known size +// ending at the file end, or unknown when live), Info, Tracks and Clusters; no +// SeekHead and no Cues. +func mkvNoSeekHead(size int, live bool) []byte { + ebml := append([]byte{0x1A, 0x45, 0xDF, 0xA3, 0x8B, 0x42, 0x82, 0x88}, "matroska"...) + out := append([]byte{}, ebml...) + out = append(out, 0x18, 0x53, 0x80, 0x67) + segStart := len(out) + 8 + if live { + out = append(out, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF) + } else { + out = append(out, ebmlSize(size-segStart)...) + } + out = append(out, 0x15, 0x49, 0xA9, 0x66, 0x84, 0, 0, 0, 0) // Info + out = append(out, 0x16, 0x54, 0xAE, 0x6B, 0x84, 0, 0, 0, 0) // Tracks + fill := size - len(out) - 12 + out = append(out, 0x1F, 0x43, 0xB6, 0x75, 0x10, 0, byte(fill>>16), byte(fill>>8)) // Cluster, 4-byte size + out[len(out)-4] = 0x10 | byte(fill>>24&0x0F) + out[len(out)-3] = byte(fill >> 16) + out[len(out)-2] = byte(fill >> 8) + out = append(out, byte(fill)) + out = append(out, bytes.Repeat([]byte{0x5A}, size-len(out))...) + return out +} + +func TestStructureVariants_Matroska(t *testing.T) { + t.Run("SeekHead with Cues at the tail (mkvmerge style)", func(t *testing.T) { requireAccepted(t, mkvPlain(4096), "a.mkv") }) + t.Run("no SeekHead, known Segment size (streaming mux)", func(t *testing.T) { requireAccepted(t, mkvNoSeekHead(8192, false), "a.mkv") }) + t.Run("no SeekHead, unknown Segment size (live webm)", func(t *testing.T) { requireAccepted(t, mkvNoSeekHead(8192, true), "a.webm") }) + t.Run("SeekHead after a Void", func(t *testing.T) { + good := mkvPlain(4096) + // Insert a 4-byte Void in front of the SeekHead and grow the Segment. + const segHdr = 16 // EBML header (16 bytes) precedes the Segment ID + _ = segHdr + requireAccepted(t, withVoidBeforeSeekHead(t, good), "a.mkv") + }) + + t.Run("head intact, Cues at the tail broken", func(t *testing.T) { + bad := mkvPlain(4096) + bad[len(bad)-9] ^= 0xFF + requireRejected(t, bad, "a.mkv") + }) + t.Run("known size claims more than the file holds, no SeekHead", func(t *testing.T) { + bad := mkvNoSeekHead(8192, false) + bad[27] += 9 // low byte of the Segment size + requireRejected(t, bad, "a.mkv") + }) + t.Run("garbage where the first Segment child belongs", func(t *testing.T) { + bad := mkvNoSeekHead(8192, true) + copy(bad[16+4+8:], []byte{0x7F, 0x7F, 0x7F, 0x7F}) + requireRejected(t, bad, "a.mkv") + }) +} + +// withVoidBeforeSeekHead rewrites mkvPlain's file with a Void element between +// the Segment header and the SeekHead (Segment size and Seek positions are +// relative to the Segment data, so every position moves by the Void's size). +func withVoidBeforeSeekHead(t *testing.T, plain []byte) []byte { + t.Helper() + const voidLen = 6 // EC 84 + 4 bytes + segDataStart := 16 + 12 + out := append([]byte{}, plain[:segDataStart]...) + out = append(out, 0xEC, 0x84, 0, 0, 0, 0) + out = append(out, plain[segDataStart:]...) + // Segment size grows by voidLen. + sz := int(plain[16+8])<<24 | int(plain[16+9])<<16 | int(plain[16+10])<<8 | int(plain[16+11]) + sz += voidLen + out[16+8], out[16+9], out[16+10], out[16+11] = byte(sz>>24), byte(sz>>16), byte(sz>>8), byte(sz) + // Seek positions of mkvPlain are u32 values at fixed places inside the SeekHead. + for i := segDataStart + voidLen; i+6 < segDataStart+voidLen+2*17+5; i++ { + if out[i] == 0x53 && out[i+1] == 0xAC && out[i+2] == 0x84 { + pos := int(out[i+3])<<24 | int(out[i+4])<<16 | int(out[i+5])<<8 | int(out[i+6]) + pos += voidLen + out[i+3], out[i+4], out[i+5], out[i+6] = byte(pos>>24), byte(pos>>16), byte(pos>>8), byte(pos) + } + } + return out +} + +// ---- MP4 ---- + +func mp4Chain(boxes ...[]byte) []byte { return bytes.Join(boxes, nil) } + +func mp4Fragmented(pairs int, lastMdat int, mfra bool) []byte { + out := mp4Chain(mp4Box("ftyp", 16), mp4Box("moov", 300)) + for i := 0; i < pairs; i++ { + out = append(out, mp4Box("moof", 40)...) + n := 200 + if i == pairs-1 { + n = lastMdat + } + out = append(out, mp4Box("mdat", n)...) + } + if mfra { + out = append(out, mp4Box("mfra", 24)...) + } + return out +} + +func TestStructureVariants_MP4(t *testing.T) { + base := func() []byte { return mp4Chain(mp4Box("ftyp", 16), mp4Box("moov", 200), mp4Box("mdat", 600)) } + + t.Run("plain moov-then-mdat", func(t *testing.T) { requireAccepted(t, base(), "a.mp4") }) + t.Run("moov at the end (not faststart)", func(t *testing.T) { + requireAccepted(t, mp4Chain(mp4Box("ftyp", 16), mp4Box("mdat", 600), mp4Box("moov", 200)), "a.mp4") + }) + t.Run("trailing free, skip and uuid boxes", func(t *testing.T) { + requireAccepted(t, mp4Chain(base(), mp4Box("free", 100), mp4Box("skip", 30), mp4Box("uuid", 40)), "a.mp4") + }) + t.Run("zero-sized last box runs to the end of the file", func(t *testing.T) { + mdat := mp4Box("mdat", 600) + copy(mdat[:4], []byte{0, 0, 0, 0}) + requireAccepted(t, mp4Chain(mp4Box("ftyp", 16), mp4Box("moov", 200), mdat), "a.mp4") + }) + t.Run("64-bit box size", func(t *testing.T) { + n := 16 + 600 + mdat := append([]byte{0, 0, 0, 1, 'm', 'd', 'a', 't', 0, 0, 0, 0, 0, 0, byte(n >> 8), byte(n)}, make([]byte, 600)...) + requireAccepted(t, mp4Chain(mp4Box("ftyp", 16), mp4Box("moov", 200), mdat), "a.mp4") + }) + t.Run("zero padding after the last box", func(t *testing.T) { + requireAccepted(t, append(base(), make([]byte, 5000)...), "a.mp4") + }) + t.Run("fragmented: thousands of moof/mdat pairs, no mfra", func(t *testing.T) { + requireAccepted(t, mp4Fragmented(3000, 200, false), "a.mp4") + }) + t.Run("fragmented with an mfra at the end", func(t *testing.T) { + requireAccepted(t, mp4Fragmented(3000, 200, true), "a.mp4") + }) + t.Run("fragmented: last fragment larger than the first tail window", func(t *testing.T) { + requireAccepted(t, mp4Fragmented(20, 6<<20, false), "a.mp4") + }) + + t.Run("non-zero junk after the last box", func(t *testing.T) { + requireRejected(t, append(base(), 1, 2, 3), "a.mp4") + }) + t.Run("zero padding that is not padding (junk follows)", func(t *testing.T) { + requireRejected(t, append(append(base(), make([]byte, 100)...), 7), "a.mp4") + }) + t.Run("no moov", func(t *testing.T) { requireRejected(t, mp4Chain(mp4Box("ftyp", 16), mp4Box("mdat", 800)), "a.mp4") }) + t.Run("last box runs past the end", func(t *testing.T) { + b := base() + requireRejected(t, b[:len(b)-50], "a.mp4") + }) + t.Run("fragmented, head fine, tail is garbage", func(t *testing.T) { + f := mp4Fragmented(3000, 200, false) + for i := len(f) - 5000; i < len(f); i++ { + f[i] = byte(i * 31) + } + requireRejected(t, f, "a.mp4") + }) + t.Run("fragmented, truncated in the middle of a box", func(t *testing.T) { + f := mp4Fragmented(3000, 200, false) + requireRejected(t, f[:len(f)-100], "a.mp4") + }) +} + +// ---- MPEG-TS / M2TS ---- + +func tsFile(stride, syncOff, packets int) []byte { + ts := make([]byte, stride*packets) + for k := 0; k < packets; k++ { + ts[k*stride+syncOff] = 0x47 + } + return ts +} + +func TestStructureVariants_TransportStreams(t *testing.T) { + for name, tc := range map[string]struct { + stride, sync int + file string + }{"188-byte TS": {188, 0, "a.ts"}, "192-byte M2TS (timestamp first)": {192, 4, "a.m2ts"}, "204-byte TS (Reed-Solomon)": {204, 0, "a.ts"}} { + t.Run(name, func(t *testing.T) { + good := tsFile(tc.stride, tc.sync, 60) + requireAccepted(t, good, tc.file) + bad := append([]byte{}, good...) + bad[59*tc.stride+tc.sync] = 0 // the last packet lost sync + requireRejected(t, bad, tc.file) + }) + } + t.Run("trailing partial packet is not a loss of sync", func(t *testing.T) { + requireAccepted(t, append(tsFile(188, 0, 60), make([]byte, 100)...), "a.ts") + }) + t.Run("capture that starts mid-packet is read as an unknown container", func(t *testing.T) { + requireAccepted(t, append(make([]byte, 77), tsFile(188, 0, 60)...), "a.ts") + }) +} diff --git a/internal/peernet/bundleimport/tracker.go b/internal/peernet/bundleimport/tracker.go new file mode 100644 index 000000000..debae0939 --- /dev/null +++ b/internal/peernet/bundleimport/tracker.go @@ -0,0 +1,311 @@ +package bundleimport + +import ( + "context" + "log/slog" + "time" + + "github.com/kipsilabs/altmount/internal/database" +) + +// Tracker methods: the state machine of an accepted import. They are the +// read and write side of "from the peer network, full check pending". + +// PendingFullCheck returns which of paths are peer-bundle files whose full +// check has not passed. The health worker forces a full check for them and +// never gives them peer advice. It fails closed: on a database error every +// path is reported pending. +func (i *Importer) PendingFullCheck(ctx context.Context, paths []string) map[string]struct{} { + rows, err := i.cfg.State.PendingAmong(ctx, paths) + if err != nil { + slog.WarnContext(ctx, "peer bundle state unreadable; treating files as pending", "error", err) + all := make(map[string]struct{}, len(paths)) + for _, p := range paths { + all[p] = struct{}{} + } + return all + } + out := make(map[string]struct{}, len(rows)) + for p := range rows { + out[p] = struct{}{} + } + return out +} + +// IsPendingPeerImport reports whether the file is a peer import with its full +// check pending. It fails open: a state error says false. +func (i *Importer) IsPendingPeerImport(ctx context.Context, filePath string) bool { + if i.cfg.State == nil { + return false + } + row, ok, err := i.cfg.State.Get(ctx, filePath) + return err == nil && ok && row.State == "pending" +} + +// PeerBundleStatus is the read method the publisher uses: fromPeer says the +// file was imported from a peer bundle, fullCheckPassed that every check it +// owed passed. A file may be published when !fromPeer || fullCheckPassed; an +// error must be treated as "not eligible". +func (i *Importer) PeerBundleStatus(ctx context.Context, filePath string) (fromPeer, fullCheckPassed bool, err error) { + row, ok, err := i.cfg.State.Get(ctx, filePath) + if err != nil || !ok { + return false, false, err + } + return true, row.State == "passed" && row.Committed, nil +} + +// AttachQueue links freshly written bundle files to the queue item that +// carries the import (direct imports learn their queue id only afterwards). +func (i *Importer) AttachQueue(ctx context.Context, paths []string, queueID int64) error { + return i.cfg.State.SetQueue(ctx, paths, queueID) +} + +// QueueHasPending reports whether a queue item still awaits its full checks, +// which is when its ARR announcement must be held back. +func (i *Importer) QueueHasPending(ctx context.Context, queueID int64) bool { + if queueID <= 0 { + return false + } + has, err := i.cfg.State.QueueHasPending(ctx, queueID) + if err != nil { + slog.WarnContext(ctx, "peer bundle state unreadable; holding the ARR announcement", "error", err) + return true + } + return has +} + +// FullCheckPassed records that a file's full check passed. When it was the +// last file of its import, the ARRs are told, exactly once. +func (i *Importer) FullCheckPassed(ctx context.Context, filePath string) { + won, err := i.cfg.State.MarkPassed(ctx, filePath) + if err != nil || !won { + if err != nil { + slog.WarnContext(ctx, "could not record a passed full check", "error", err) + } + return + } + i.cnt.add(func(c *counters) { c.passed++ }) + row, ok, err := i.cfg.State.Get(ctx, filePath) + if err != nil || !ok { + return + } + i.announce(ctx, row.QueueID) + if !row.ArrOwed && !i.QueueHasPending(ctx, row.QueueID) { + // No ARR is told about this import (the caller suppressed it, or it is a + // direct import), so nothing else lifts the hold. + if fh, ok := i.cfg.Announcer.(FailureHandler); ok && row.QueueID > 0 { + _ = fh.ReleaseHold(ctx, row.QueueID) + } + } + if v, ok := i.cfg.Reporter.(BundleVerifiedReporter); ok && !i.QueueHasPending(ctx, row.QueueID) { + v.ReportVerifiedBundle(ctx, Origin{NetworkID: row.NetworkID, PublisherID: row.PublisherID}, row.ReleaseID) + } +} + +func (i *Importer) announce(ctx context.Context, queueID int64) { + if queueID <= 0 || i.cfg.Announcer == nil { + return + } + row, claimed, err := i.cfg.State.ClaimAnnouncement(ctx, queueID) + if err != nil || !claimed { + return + } + err = i.cfg.Announcer.AnnounceVerifiedImport(ctx, Announcement{ + QueueID: queueID, Category: row.Category, TargetPath: row.TargetPath, ResultingPath: row.ResultingPath, + }) + if err != nil { + slog.WarnContext(ctx, "could not announce a verified peer import; will retry", "queue_id", queueID, "error", err) + _ = i.cfg.State.ReleaseAnnouncement(ctx, queueID) + return + } + i.cnt.add(func(c *counters) { c.announce++ }) +} + +// FullCheckFailed records that a file's full check failed: the whole import is +// failed (see failImport), it is never announced or published, and the origin publisher is reported once per +// release as having served a bad bundle. +func (i *Importer) FullCheckFailed(ctx context.Context, filePath string) { + row, ok, err := i.cfg.State.Get(ctx, filePath) + if err != nil || !ok { + return + } + won, err := i.cfg.State.MarkFailed(ctx, filePath) + if err != nil || !won { + return + } + i.cnt.add(func(c *counters) { c.failed++ }) + i.failImport(ctx, row, failReasonCheck) + if i.cfg.Reporter == nil { + return + } + i.reportedMu.Lock() + first := i.reported == nil || !i.reported[row.ReleaseID+"/"+row.PublisherID] + if i.reported == nil || len(i.reported) >= maxReported { + // Bounded: forgetting old entries only means a publisher may be + // reported once more for a release, which is harmless. + i.reported = map[string]bool{} + } + i.reported[row.ReleaseID+"/"+row.PublisherID] = true + i.reportedMu.Unlock() + if !first { + return + } + i.cfg.Reporter.ReportBadBundle(ctx, Origin{NetworkID: row.NetworkID, PublisherID: row.PublisherID}, row.ReleaseID) + i.cnt.add(func(c *counters) { c.reports++ }) +} + +// Messages a failed peer import carries in the SABnzbd-compatible history. +const ( + failReasonCheck = "Peer-supplied release failed its full health check and was removed" + failReasonStale = "Peer-supplied release was not fully verified within 7 days and was removed" +) + +// failImport ends a peer import that failed its full check, or never got one: +// the queue item becomes a failed one (the ARR that grabbed it sees it in the +// history, blocklists the release and searches again; a direct import has no +// ARR), then every file of the import is removed (metadata, content index, +// library copy, health record) and its state rows deleted, so a corrupt file +// does not stay in the library and the surviving files of the release cannot +// complete the import by passing later. It touches only the rows of this +// import, and every step is idempotent. +func (i *Importer) failImport(ctx context.Context, row database.PeernetBundleImport, reason string) { + bg := context.WithoutCancel(ctx) + fh, _ := i.cfg.Announcer.(FailureHandler) + rows := []database.PeernetBundleImport{row} + if row.QueueID > 0 { + if l, err := i.cfg.State.ListByQueue(bg, row.QueueID); err == nil && len(l) > 0 { + rows = l + } + if fh != nil { + if err := fh.FailImport(bg, row.QueueID, reason); err != nil { + slog.WarnContext(ctx, "could not mark a failed peer import as failed in the queue", "queue_id", row.QueueID, "error", err) + } + } + } + paths := make([]string, 0, len(rows)) + for _, r := range rows { + paths = append(paths, r.FilePath) + var err error + if fh != nil { + err = fh.RemoveFailedFile(bg, r.FilePath) + } else { + err = i.cfg.Library.DeleteFileMetadata(bg, r.FilePath) + } + if err != nil { + slog.WarnContext(ctx, "could not remove a file of a failed peer import", "file_path", r.FilePath, "error", err) + } + if i.cfg.Index != nil { + _ = i.cfg.Index.Delete(bg, r.FilePath) + } + } + if err := i.cfg.State.Delete(bg, paths); err != nil { + slog.WarnContext(ctx, "could not drop the state of a failed peer import", "error", err) + } +} + +// releaseOrphanHolds lifts the hold of every queue item that no state row +// refers to any more. A hold nothing can release would keep the item in the +// ARR-visible queue forever. +func (i *Importer) releaseOrphanHolds(ctx context.Context) { + fh, ok := i.cfg.Announcer.(FailureHandler) + if !ok || i.cfg.State == nil { + return + } + ids, err := i.cfg.State.OrphanedHolds(ctx) + if err != nil { + slog.WarnContext(ctx, "could not list orphaned peer holds", "error", err) + return + } + for _, id := range ids { + if err := fh.ReleaseHold(ctx, id); err != nil { + slog.WarnContext(ctx, "could not release an orphaned peer hold", "queue_id", id, "error", err) + } + } +} + +// maxReported bounds the set of (release, publisher) pairs already reported. +const maxReported = 4096 + +// MaxPendingAge is how long a peer import may owe its full check. After it the +// import is failed (never announced, never published) instead of waiting +// forever, for example because health checking was turned off. +const MaxPendingAge = 7 * 24 * time.Hour + +// PendingStatus returns how many files owe their full check and how many of +// those are older than MaxPendingAge. +func (i *Importer) PendingStatus(ctx context.Context) (pending, stale int) { + if i.cfg.State == nil { + return 0, 0 + } + p, s, err := i.cfg.State.PendingCounts(ctx, i.cfg.Now().Add(-MaxPendingAge)) + if err != nil { + return 0, 0 + } + return p, s +} + +// EscalateStale fails the pending files older than MaxPendingAge and returns how +// many. The hold on the ARRs is never lifted for them and they are never +// published; the publisher is not reported, since no check ever failed. +func (i *Importer) EscalateStale(ctx context.Context) int { + if i.cfg.State == nil { + return 0 + } + rows, err := i.cfg.State.StalePending(ctx, i.cfg.Now().Add(-MaxPendingAge), 500) + if err != nil { + slog.WarnContext(ctx, "could not list stale peer imports", "error", err) + return 0 + } + n := 0 + for _, r := range rows { + if won, err := i.cfg.State.MarkFailed(ctx, r.FilePath); err == nil && won { + n++ + i.failImport(ctx, r, failReasonStale) + } + } + if n > 0 { + i.cnt.add(func(c *counters) { c.failed += uint64(n) }) + slog.WarnContext(ctx, "peer imports never got their full check and were failed", "files", n) + } + return n +} + +// RunHousekeeping escalates stale imports once at start and then hourly until +// ctx ends. +func (i *Importer) RunHousekeeping(ctx context.Context) { + // A background task must never take the process down. + defer func() { + if r := recover(); r != nil { + slog.ErrorContext(ctx, "peer import housekeeping stopped", "panic", r) + } + }() + i.Reconcile(ctx) + i.EscalateStale(ctx) + t := time.NewTicker(time.Hour) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + i.EscalateStale(ctx) + i.releaseOrphanHolds(ctx) + } + } +} + +// ResumeAnnouncements delivers announcements whose files all passed but that a +// restart interrupted. Call it once at start-up. +func (i *Importer) ResumeAnnouncements(ctx context.Context) { + if i.cfg.State == nil || i.cfg.Announcer == nil { + return + } + ids, err := i.cfg.State.QueuesAwaitingAnnouncement(ctx) + if err != nil { + slog.WarnContext(ctx, "could not list pending peer announcements", "error", err) + return + } + for _, id := range ids { + i.announce(ctx, id) + } +} diff --git a/internal/peernet/bundleimport/tracker_test.go b/internal/peernet/bundleimport/tracker_test.go new file mode 100644 index 000000000..636b50799 --- /dev/null +++ b/internal/peernet/bundleimport/tracker_test.go @@ -0,0 +1,390 @@ +package bundleimport + +import ( + "context" + "errors" + "os/exec" + "path" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func (e *env) accept(tag string, specs ...fileSpec) (*bundle.Bundle, *Result) { + b := e.mkBundle(tag, specs...) + res, err := e.imp.Accept(context.Background(), e.req(b)) + require.NoError(e.t, err) + return b, res +} + +func TestTracker_PendingAndPublishEligibility(t *testing.T) { + e := newEnv(t) + _, res := e.accept("tr", mkvSpec("movie.mkv")) + ctx := context.Background() + p := res.Paths[0] + + from, passed, err := e.imp.PeerBundleStatus(ctx, p) + require.NoError(t, err) + assert.True(t, from) + assert.False(t, passed, "not publishable until the full check passed") + assert.Contains(t, e.imp.PendingFullCheck(ctx, []string{p, "other.mkv"}), p) + assert.NotContains(t, e.imp.PendingFullCheck(ctx, []string{"other.mkv"}), "other.mkv") + + from, passed, err = e.imp.PeerBundleStatus(ctx, "movies/local-import.mkv") + require.NoError(t, err) + assert.False(t, from, "a normal import is not from a peer") + assert.False(t, passed) + + e.imp.FullCheckPassed(ctx, p) + from, passed, _ = e.imp.PeerBundleStatus(ctx, p) + assert.True(t, from) + assert.True(t, passed, "publish-eligible once passed") + assert.Empty(t, e.imp.PendingFullCheck(ctx, []string{p})) +} + +func TestTracker_ARRAnnouncedOnceAfterLastFilePasses(t *testing.T) { + e := newEnv(t) + _, res := e.accept("arr", mkvSpec("a.mkv"), mkvSpec("b.mkv")) + ctx := context.Background() + require.Len(t, res.Paths, 2) + + e.imp.FullCheckPassed(ctx, res.Paths[0]) + assert.Empty(t, e.ann.got, "one file still pending: nothing announced") + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { defer wg.Done(); e.imp.FullCheckPassed(ctx, res.Paths[1]) }() + } + wg.Wait() + e.imp.FullCheckPassed(ctx, res.Paths[1]) + require.Len(t, e.ann.got, 1, "exactly once") + assert.Equal(t, int64(11), e.ann.got[0].QueueID) + assert.Equal(t, "movies", e.ann.got[0].Category) + assert.Equal(t, "movies/Test.Release", e.ann.got[0].ResultingPath) +} + +func TestTracker_GateFailsClosedAcrossRestart(t *testing.T) { + e := newEnv(t) + _, res := e.accept("restart", mkvSpec("a.mkv")) + ctx := context.Background() + + // "Restart": a brand new importer over the same database. + imp2 := New(Config{Library: e.lib, State: e.repo, Announcer: e.ann, HealthEnabled: func() bool { return true }}) + assert.Contains(t, imp2.PendingFullCheck(ctx, res.Paths), res.Paths[0], "still pending after restart") + assert.True(t, imp2.QueueHasPending(ctx, 11)) + imp2.ResumeAnnouncements(ctx) + assert.Empty(t, e.ann.got, "no announcement while pending") + + // The pass was recorded but the process died before the ARRs were told. + _, err := e.repo.MarkPassed(ctx, res.Paths[0]) + require.NoError(t, err) + imp3 := New(Config{Library: e.lib, State: e.repo, Announcer: e.ann}) + imp3.ResumeAnnouncements(ctx) + imp3.ResumeAnnouncements(ctx) + assert.Len(t, e.ann.got, 1, "delivered once after restart") +} + +func TestTracker_FailedAnnouncementIsRetried(t *testing.T) { + e := newEnv(t) + _, res := e.accept("retry", mkvSpec("a.mkv")) + var calls atomic.Int32 + e.imp.cfg.Announcer = announcerFunc(func(Announcement) error { + if calls.Add(1) == 1 { + return errors.New("arr down") + } + return nil + }) + ctx := context.Background() + e.imp.FullCheckPassed(ctx, res.Paths[0]) + e.imp.ResumeAnnouncements(ctx) + assert.Equal(t, int32(2), calls.Load()) + e.imp.ResumeAnnouncements(ctx) + assert.Equal(t, int32(2), calls.Load(), "no third delivery") +} + +type announcerFunc func(Announcement) error + +func (f announcerFunc) AnnounceVerifiedImport(_ context.Context, a Announcement) error { return f(a) } + +func TestTracker_FailedFullCheckReportsPublisherAndNeverAnnounces(t *testing.T) { + e := newEnv(t) + b, res := e.accept("bad", mkvSpec("a.mkv"), mkvSpec("b.mkv")) + ctx := context.Background() + e.imp.FullCheckPassed(ctx, res.Paths[0]) + e.imp.FullCheckFailed(ctx, res.Paths[1]) + e.imp.FullCheckFailed(ctx, res.Paths[0]) // passed already: no-op + + assert.Equal(t, []string{strings.Repeat("d", 64) + "/" + b.ReleaseID.String()}, e.rep.got) + assert.Empty(t, e.ann.got) + _, passed, _ := e.imp.PeerBundleStatus(ctx, res.Paths[1]) + assert.False(t, passed, "a failed file never becomes publish-eligible") + c := e.imp.Counters() + assert.Equal(t, uint64(1), c.FullCheckFailed) + assert.Equal(t, uint64(1), c.FullCheckPassed) + assert.Equal(t, uint64(1), c.BadBundleReports) +} + +func TestTracker_CallerSuppressedARRIsNeverAnnounced(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("skip", mkvSpec("a.mkv")) + req := e.req(b) + req.Target.SkipArr = true + res, err := e.imp.Accept(context.Background(), req) + require.NoError(t, err) + e.imp.FullCheckPassed(context.Background(), res.Paths[0]) + assert.Empty(t, e.ann.got) +} + +type fakeFetcher struct { + b *bundle.Bundle + err error + calls atomic.Int32 + gate chan struct{} +} + +func (f *fakeFetcher) FetchBundle(ctx context.Context, _ string) (*bundle.Bundle, Origin, error) { + f.calls.Add(1) + if f.gate != nil { + <-f.gate + } + return f.b, Origin{NetworkID: strings.Repeat("c", 64), PublisherID: strings.Repeat("d", 64)}, f.err +} + +type fakeRecorder struct { + mu sync.Mutex + recs []ImportRecord +} + +func (r *fakeRecorder) RecordPeerImport(_ context.Context, rec ImportRecord) (int64, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.recs = append(r.recs, rec) + return 77, nil +} + +func TestImportRelease_FetchAcceptAndRecord(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("ir", mkvSpec("movie.mkv")) + rec := &fakeRecorder{} + e.imp.cfg.Fetcher = &fakeFetcher{b: b} + e.imp.cfg.Recorder = rec + res, err := e.imp.ImportRelease(context.Background(), b.ReleaseID.String(), Target{Category: "movies"}) + require.NoError(t, err) + assert.Equal(t, "movies/Test Release ir", res.Dir, "folder is category plus the sanitised bundle title") + require.Len(t, rec.recs, 1) + assert.Equal(t, "Test Release ir", rec.recs[0].Title) + assert.Equal(t, res.Paths, rec.recs[0].WrittenPaths) + assert.Equal(t, b.ReleaseID.String(), rec.recs[0].ReleaseID) + assert.True(t, e.imp.QueueHasPending(context.Background(), 77), "rows attached to the new queue item") +} + +func TestImportRelease_IdempotentAndCoalesced(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("co", mkvSpec("movie.mkv")) + f := &fakeFetcher{b: b, gate: make(chan struct{})} + rec := &fakeRecorder{} + e.imp.cfg.Fetcher, e.imp.cfg.Recorder = f, rec + var wg sync.WaitGroup + for i := 0; i < 5; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, err := e.imp.ImportRelease(context.Background(), b.ReleaseID.String(), Target{Category: "movies"}) + assert.NoError(t, err) + }() + } + for f.calls.Load() == 0 { + } + close(f.gate) + wg.Wait() + assert.Equal(t, int32(1), f.calls.Load(), "concurrent requests coalesce into one fetch") + assert.Len(t, rec.recs, 1) + + res, err := e.imp.ImportRelease(context.Background(), b.ReleaseID.String(), Target{Category: "movies"}) + require.NoError(t, err) + assert.True(t, res.AlreadyImported) + assert.Len(t, rec.recs, 1, "an already imported release is not recorded twice") +} + +func TestImportRelease_FailuresLeaveNoRecord(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("fail", mkvSpec("movie.mkv")) + rec := &fakeRecorder{} + e.imp.cfg.Recorder = rec + e.imp.cfg.Fetcher = &fakeFetcher{err: errors.New("no peer")} + _, err := e.imp.ImportRelease(context.Background(), b.ReleaseID.String(), Target{Category: "movies"}) + assert.Equal(t, ReasonUnavailable, ReasonOf(err)) + + e.imp.cfg.Fetcher = nil + _, err = e.imp.ImportRelease(context.Background(), b.ReleaseID.String(), Target{Category: "movies"}) + assert.Equal(t, ReasonNoNetwork, ReasonOf(err)) + + e.prober.err = ErrProbeMissing + e.imp.cfg.Fetcher = &fakeFetcher{b: b} + _, err = e.imp.ImportRelease(context.Background(), b.ReleaseID.String(), Target{Category: "movies"}) + assert.Equal(t, ReasonProbeMissing, ReasonOf(err)) + assert.Empty(t, rec.recs) + e.assertNothingBehind() +} + +func TestImportRelease_RejectsABundleOfAnotherRelease(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("one", mkvSpec("movie.mkv")) + other := e.mkBundle("two", mkvSpec("movie.mkv")) + e.imp.cfg.Fetcher = &fakeFetcher{b: other} + e.imp.cfg.Recorder = &fakeRecorder{} + _, err := e.imp.ImportRelease(context.Background(), b.ReleaseID.String(), Target{Category: "movies"}) + assert.Equal(t, ReasonInvalid, ReasonOf(err)) + e.assertNothingBehind() +} + +// No archive analysis and no key derivation may be reachable from this +// package: the layout is already resolved. The serving stack's rclone cipher +// does link a KDF, but only for rclone-encrypted files, which a bundle cannot +// carry; so the KDF ban is on this package's own imports. +func TestPackageNeverImportsArchiveAnalysisOrKDF(t *testing.T) { + deps, err := exec.Command("go", "list", "-deps", ".").Output() + require.NoError(t, err) + for _, dep := range strings.Fields(string(deps)) { + for _, banned := range []string{"internal/importer/archive", "rardecode", "sevenzip", "internal/importer/multifile", "internal/importer/singlefile"} { + assert.NotContains(t, dep, banned) + } + } + direct, err := exec.Command("go", "list", "-f", "{{join .Imports \"\\n\"}}", ".").Output() + require.NoError(t, err) + for _, imp := range strings.Fields(string(direct)) { + assert.NotContains(t, imp, "x/crypto") + assert.NotContains(t, imp, "pbkdf2") + assert.NotContains(t, imp, "scrypt") + } +} + +func TestTracker_VerifiedBundleIsReportedOnceEveryFilePassed(t *testing.T) { + e := newEnv(t) + b, res := e.accept("good", mkvSpec("a.mkv"), mkvSpec("b.mkv")) + ctx := context.Background() + e.imp.FullCheckPassed(ctx, res.Paths[0]) + assert.Empty(t, e.rep.verified, "one file still pending") + e.imp.FullCheckPassed(ctx, res.Paths[1]) + assert.Equal(t, []string{strings.Repeat("d", 64) + "/" + b.ReleaseID.String()}, e.rep.verified) + assert.Empty(t, e.rep.got) +} + +func TestImportRelease_PassesMediaMetadataAndDownloadIDToTheRecorder(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("md", mkvSpec("movie.mkv")) + rec := &fakeRecorder{} + e.imp.cfg.Fetcher, e.imp.cfg.Recorder = &fakeFetcher{b: b}, rec + _, err := e.imp.ImportRelease(context.Background(), b.ReleaseID.String(), + Target{Category: "movies", Metadata: `{"imdb_id":"tt1"}`, DownloadID: "stremio:x"}) + require.NoError(t, err) + require.Len(t, rec.recs, 1) + assert.Equal(t, `{"imdb_id":"tt1"}`, rec.recs[0].Metadata) + assert.Equal(t, "stremio:x", rec.recs[0].DownloadID) +} + +func TestImportRelease_DirFuncChoosesTheFolderFromTheSanitisedTitle(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("df", mkvSpec("movie.mkv")) + b.Title = "Some Title (2020)" + e.imp.cfg.Fetcher, e.imp.cfg.Recorder = &fakeFetcher{b: b}, &fakeRecorder{} + var got string + res, err := e.imp.ImportRelease(context.Background(), b.ReleaseID.String(), Target{Category: "movies", + DirFunc: func(title string) string { got = title; return "complete/movies/" + title }}) + require.NoError(t, err) + assert.Equal(t, bundle.SanitizeFilename(b.Title), got, "the function only ever sees the sanitised title") + assert.Equal(t, "complete/movies/"+got, res.Dir) +} + +func TestImportRelease_HostileTitlesNeverEscapeTheCategoryFolder(t *testing.T) { + for _, title := range []string{"...mkv", ".mkv", ".hidden", "patches"} { + e := newEnv(t) + b := e.mkBundle("ht", mkvSpec("movie.mkv")) + b.Title = title + e.imp.cfg.Fetcher, e.imp.cfg.Recorder = &fakeFetcher{b: b}, &fakeRecorder{} + // Without a DirFunc the default folder is category plus title. + res, err := e.imp.ImportRelease(context.Background(), b.ReleaseID.String(), Target{Category: "movies"}) + if err == nil { + assert.Equal(t, "movies", path.Dir(res.Dir), "title %q: %q", title, res.Dir) + assert.NotContains(t, []string{".", "..", ""}, path.Base(res.Dir), title) + } + require.Error(t, err, "title %q must be refused", title) + assert.Equal(t, ReasonBadTarget, ReasonOf(err), title) + // A DirFunc that refuses the title (returns "") never falls back. + e2 := newEnv(t) + b2 := e2.mkBundle("ht2", mkvSpec("movie.mkv")) + b2.Title = title + e2.imp.cfg.Fetcher, e2.imp.cfg.Recorder = &fakeFetcher{b: b2}, &fakeRecorder{} + _, err = e2.imp.ImportRelease(context.Background(), b2.ReleaseID.String(), Target{Category: "movies", DirFunc: func(string) string { return "" }}) + assert.Equal(t, ReasonBadTarget, ReasonOf(err), title) + } +} + +func TestCleanTargetRefusesReservedAndCategoryOnlyFolders(t *testing.T) { + for _, dir := range []string{"movies", "complete/.nzbs/x", "movies/patches/x", "movies/.peernet-staging", "movies/Corrupted_Metadata"} { + _, err := cleanTarget(dir) + assert.Error(t, err, dir) + } + got, err := cleanTarget("complete/movies/Some Movie") + require.NoError(t, err) + assert.Equal(t, "complete/movies/Some Movie", got) +} + +func TestEscalateStale_FailsOldPendingImportsAndNeverAnnounces(t *testing.T) { + e := newEnv(t) + _, res := e.accept("st", mkvSpec("a.mkv")) + ctx := context.Background() + require.NoError(t, e.repo.SetQueue(ctx, res.Paths, 11)) + + pending, stale := e.imp.PendingStatus(ctx) + assert.Equal(t, 1, pending) + assert.Zero(t, stale, "a fresh import is not stale") + + e.imp.cfg.Now = func() time.Time { return time.Now().Add(MaxPendingAge + time.Hour) } + _, stale = e.imp.PendingStatus(ctx) + assert.Equal(t, 1, stale) + assert.Equal(t, 1, e.imp.EscalateStale(ctx)) + + _, ok, _ := e.repo.Get(ctx, res.Paths[0]) + assert.False(t, ok, "a failed import is removed with its state rows") + assert.False(t, e.lib.FileExists(res.Paths[0]), "and so are its files") + assert.Empty(t, e.ann.got, "an unverified import is never announced") + assert.Empty(t, e.imp.PendingFullCheck(ctx, res.Paths), "no longer pending") + assert.Empty(t, e.rep.got, "the publisher is not blamed for a check that never ran") +} + +func TestImportRelease_FirstCallerLeavingDoesNotFailTheOthers(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("sf", mkvSpec("movie.mkv")) + gate := make(chan struct{}) + e.imp.cfg.Fetcher, e.imp.cfg.Recorder = &fakeFetcher{b: b, gate: gate}, &fakeRecorder{} + first, leave := context.WithCancel(context.Background()) + errFirst := make(chan error, 1) + go func() { + _, err := e.imp.ImportRelease(first, b.ReleaseID.String(), Target{Category: "movies"}) + errFirst <- err + }() + time.Sleep(50 * time.Millisecond) + type out struct { + res *Result + err error + } + second := make(chan out, 1) + go func() { + r, err := e.imp.ImportRelease(context.Background(), b.ReleaseID.String(), Target{Category: "movies"}) + second <- out{r, err} + }() + time.Sleep(50 * time.Millisecond) + leave() + close(gate) + o := <-second + require.NoError(t, o.err, "a follower must not inherit the first caller's cancellation") + require.NotNil(t, o.res) + <-errFirst +} diff --git a/internal/peernet/bundleimport/types.go b/internal/peernet/bundleimport/types.go new file mode 100644 index 000000000..8d71207ef --- /dev/null +++ b/internal/peernet/bundleimport/types.go @@ -0,0 +1,418 @@ +// Package bundleimport accepts a verified peer bundle as a local import. +// +// A bundle is a sanitised finished import: segment lists and a resolved file +// layout. Accepting one lets this node mount a release without downloading or +// parsing archives. Everything in a bundle is hostile input, so Accept +// re-validates it, applies local policy, probes the release against the user's +// own providers, proves through the real reader stack that the start of the +// main media file decodes, and only then writes anything the user can see. +// What it writes is marked as coming from the peer network and held back from +// the ARRs and from publishing until a full health check of every file passed. +// +// No archive parser and no key derivation run anywhere in this package: the +// layout is already resolved, and a wrong AES key just yields bytes that fail +// the media probe. +// +// The package defines the narrow interfaces it needs; internal/health and +// internal/importer never import it (see internal/peernet/bridge). +package bundleimport + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/kipsilabs/altmount/internal/contentverify" + "github.com/kipsilabs/altmount/internal/database" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/bundle" +) + +// Origin names where a bundle came from. It is kept only so a bad bundle can be +// reported; it is never shown as a link. +type Origin struct { + NetworkID string + PublisherID string + // Open says the bundle came from the open network (anyone can join it). + Open bool + // Corroborators is how many distinct network ranges announced this very + // Bundle ID. + Corroborators int +} + +// Fetcher returns a bundle that is already hash- and ID-verified (bundle.Verify) +// for a Release ID, with the network and publisher it came from. It is +// implemented by the peer network service and injected later. +type Fetcher interface { + FetchBundle(ctx context.Context, releaseID string) (*bundle.Bundle, Origin, error) +} + +// BadBundleReporter is told when a publisher served a bundle whose release +// failed its full local check, so the network side can back that publisher off. +// It must return quickly and never fail. +type BadBundleReporter interface { + ReportBadBundle(ctx context.Context, origin Origin, releaseID string) +} + +// BundleVerifiedReporter is optionally implemented by a BadBundleReporter: it +// is told when every file of an import from a peer bundle passed its full +// check, so an earlier penalty of that publisher can be cleared. +type BundleVerifiedReporter interface { + ReportVerifiedBundle(ctx context.Context, origin Origin, releaseID string) +} + +// ProbeFile is one file of the release probe: every real segment id the file +// reads through (own segments and outer archive volumes), in order. +type ProbeFile struct { + Name string + SegmentIDs []string +} + +// ReleaseProber is the importer's release probe (a STAT sample against the +// user's own providers) with the same sampling rules as a normal import. It +// returns ErrProbeMissing when sampled articles are gone, ErrProbeInconclusive +// when the providers could not answer, nil when the sample is reachable. +type ReleaseProber interface { + ProbeRelease(ctx context.Context, files []ProbeFile) error +} + +var ( + // ErrProbeMissing means a sampled article does not exist on the providers. + ErrProbeMissing = errors.New("bundleimport: sampled articles are missing") + // ErrProbeInconclusive means the providers could not answer. + ErrProbeInconclusive = errors.New("bundleimport: release probe inconclusive") +) + +// Library is the local metadata store the accepted files are written to. The +// real metadata service satisfies it, so every path chokepoint and observer +// (the content index) applies. +type Library interface { + FileExists(virtualPath string) bool + ListDirectory(virtualPath string) ([]string, error) + ReadFileMetadata(virtualPath string) (*metapb.FileMetadata, error) + WriteFileMetadata(virtualPath string, m *metapb.FileMetadata) error + DeleteFileMetadata(ctx context.Context, virtualPath string) error +} + +// ContentIndex is the local content index (the database repository). Accept +// records the bundle's verified Release ID and scheme in it for the files it +// writes, because the written metadata cannot reproduce them. +type ContentIndex interface { + UpsertBatch(ctx context.Context, entries []database.PeernetContentEntry) error + Delete(ctx context.Context, filePath string) error +} + +// Stager makes a hidden, throwaway place where a bundle's layout can be read +// through the real serving stack before anything the user can see is written. +type Stager interface { + NewStaging() (Staging, error) +} + +// Staging is one hidden staging area. Nothing in it is ever listed, mounted or +// indexed; Close removes all of it. +type Staging interface { + WriteFileMetadata(virtualPath string, m *metapb.FileMetadata) error + Opener() contentverify.Opener + Close() error +} + +// State is the persisted peer-bundle bookkeeping (the database repository). +type State interface { + Upsert(ctx context.Context, rows []database.PeernetBundleImport) error + Get(ctx context.Context, filePath string) (database.PeernetBundleImport, bool, error) + Delete(ctx context.Context, paths []string) error + PendingAmong(ctx context.Context, paths []string) (map[string]database.PeernetBundleImport, error) + MarkPassed(ctx context.Context, filePath string) (bool, error) + MarkFailed(ctx context.Context, filePath string) (bool, error) + ClaimAnnouncement(ctx context.Context, queueID int64) (database.PeernetBundleImport, bool, error) + ReleaseAnnouncement(ctx context.Context, queueID int64) error + QueuesAwaitingAnnouncement(ctx context.Context) ([]int64, error) + QueueHasPending(ctx context.Context, queueID int64) (bool, error) + SetQueue(ctx context.Context, paths []string, queueID int64) error + MoveFile(ctx context.Context, from, to string) (bool, error) + MoveDir(ctx context.Context, from, to string) error + DeleteDir(ctx context.Context, dir string) error + PendingCounts(ctx context.Context, staleBefore time.Time) (pending, stale int, err error) + StalePending(ctx context.Context, staleBefore time.Time, limit int) ([]database.PeernetBundleImport, error) + CommitRows(ctx context.Context, paths []string) error + ListUncommitted(ctx context.Context, limit int) ([]database.PeernetBundleImport, error) + ListByQueue(ctx context.Context, queueID int64) ([]database.PeernetBundleImport, error) + ListAfter(ctx context.Context, afterPath string, limit int) ([]database.PeernetBundleImport, error) + OrphanedHolds(ctx context.Context) ([]int64, error) +} + +// Announcement is what the ARRs are told about once every file of a bundle +// import passed its full check. +type Announcement struct { + QueueID int64 + Category string + TargetPath string + ResultingPath string +} + +// Announcer lifts the hold on a queue item (it stops being hidden from the +// SABnzbd-compatible history) and notifies the ARRs. It is implemented by the +// importer; it is called at most once per import. +type Announcer interface { + AnnounceVerifiedImport(ctx context.Context, a Announcement) error +} + +// FailureHandler is what ends a peer import that failed its full check (or +// never got one): the Announcer implemented by the importer also provides it. +// Every method is idempotent. +type FailureHandler interface { + // FailImport turns the held queue item into a failed one with a short + // message, so the SABnzbd-compatible history reports it as failed. + FailImport(ctx context.Context, queueID int64, reason string) error + // RemoveFailedFile removes one file of a failed import the way the health + // worker removes a corrupted file: metadata, library copy and health record. + RemoveFailedFile(ctx context.Context, virtualPath string) error + // ReleaseHold lifts the hold of a queue item that no file refers to any more. + ReleaseHold(ctx context.Context, queueID int64) error +} + +// Recorder creates the queue and history entries of a direct import (one with +// no NZB) so it looks like a normal import to the rest of the system. It +// returns the queue id used to hold the ARR announcement. +type Recorder interface { + RecordPeerImport(ctx context.Context, rec ImportRecord) (queueID int64, err error) +} + +// ImportRecord describes a finished direct import. +type ImportRecord struct { + Title string + Category string + TargetPath string + ResultingPath string + WrittenPaths []string + TotalSize int64 + ReleaseID string + // SkipArr is the caller's own wish not to notify the ARRs. + SkipArr bool + // Metadata and DownloadID come from the caller's Target. + Metadata string + DownloadID string +} + +// Reason is why a bundle was rejected. +type Reason string + +// Rejection reasons. Every one has a counter. +const ( + ReasonInvalid Reason = "invalid" // structure or identity check failed + ReasonTooLarge Reason = "too_large" // over the local size limit + ReasonTooManyFiles Reason = "too_many_files" // over the local file-count limit + ReasonBadName Reason = "bad_name" // file name is not a plain sanitised name + ReasonBadTarget Reason = "bad_target" // the local target folder is unusable + ReasonCollision Reason = "collision" // a different file already sits at a target path + ReasonHealthDisabled Reason = "health_disabled" // no full check can ever run + ReasonProbeMissing Reason = "probe_missing" // release probe found missing articles + ReasonProbeInconclusive Reason = "probe_inconclusive" // release probe could not answer + ReasonNoMediaFile Reason = "no_media_file" // nothing the media probe can verify + ReasonMediaInvalid Reason = "media_invalid" // media header did not decode + ReasonMediaMissing Reason = "media_missing" // media head article missing + ReasonMediaInconclusive Reason = "media_inconclusive" // media probe could not complete + ReasonStaging Reason = "staging_failed" // hidden staging area failed + ReasonWriteFailed Reason = "write_failed" // local write failed (rolled back) + ReasonNZBMismatch Reason = "nzb_mismatch" // bundle does not match the user's NZB + ReasonUnavailable Reason = "unavailable" // no bundle could be fetched + ReasonNoNetwork Reason = "no_network" // feature not wired + ReasonSizeImplausible Reason = "size_implausible" // declared sizes do not fit the user's NZB + ReasonYencMismatch Reason = "yenc_mismatch" // sampled article headers contradict the layout + ReasonStructureInvalid Reason = "structure_invalid" // media structure check failed beyond the first bytes + ReasonForbiddenFile Reason = "forbidden_file" // a file a media server or the OS could act on + ReasonSidecarTooLarge Reason = "sidecar_too_large" // a subtitle or nfo over its size cap +) + +// RejectError is the typed rejection Accept returns. +type RejectError struct { + Reason Reason + Err error +} + +func (e *RejectError) Error() string { + if e.Err == nil { + return "bundle rejected: " + string(e.Reason) + } + return fmt.Sprintf("bundle rejected: %s: %v", e.Reason, e.Err) +} + +// Unwrap exposes the cause. +func (e *RejectError) Unwrap() error { return e.Err } + +func reject(r Reason, err error) *RejectError { return &RejectError{Reason: r, Err: err} } + +// ReasonOf returns the rejection reason of err, or "" when it is not one. +func ReasonOf(err error) Reason { + var re *RejectError + if errors.As(err, &re) { + return re.Reason + } + return "" +} + +// Policy is the local acceptance policy. +type Policy struct { + MaxTotalSize int64 + MaxFiles int + // MaxMediaProbes caps how many media files are probed through the layout. + MaxMediaProbes int + // ProbeTimeout bounds the read of one media head. + MediaProbeTimeout time.Duration + // AcceptTimeout bounds the whole of Accept. + AcceptTimeout time.Duration + // MaxYencSamples caps the articles whose yEnc headers are read, and + // YencRandomPerTable is how many random ones each segment table adds to the + // first and last. + MaxYencSamples int + YencRandomPerTable int + // MaxMediaFiles caps the media files a bundle may write (each is verified). + MaxMediaFiles int + // MaxSidecars caps the small allow-listed files (subtitles, nfo) written. + MaxSidecars int + // MaxSubtitleBytes and MaxNfoBytes cap one sidecar file. + MaxSubtitleBytes int64 + MaxNfoBytes int64 +} + +// DefaultPolicy returns the default local limits. +func DefaultPolicy() Policy { + return Policy{ + MaxTotalSize: int64(2) << 40, // 2 TiB + MaxFiles: 2_000, + MaxMediaProbes: 4, + MediaProbeTimeout: 30 * time.Second, + AcceptTimeout: 2 * time.Minute, + MaxYencSamples: 24, + YencRandomPerTable: 2, + MaxMediaFiles: 40, + MaxSidecars: 50, + MaxSubtitleBytes: 16 << 20, + MaxNfoBytes: 1 << 20, + } +} + +// MediaProbes returns the media probe cap (at least 1). +func (p Policy) MediaProbes() int { return max(p.MaxMediaProbes, 1) } + +// MediaTimeout returns the per-file media probe timeout. +func (p Policy) MediaTimeout() time.Duration { + if p.MediaProbeTimeout <= 0 { + return 30 * time.Second + } + return p.MediaProbeTimeout +} + +// Target is where an accepted bundle goes. It is decided locally by the +// caller, never taken from the bundle. +type Target struct { + // Dir is the local virtual folder the files go into (for example + // "movies/Release.Name"). Required. + Dir string + // Category is the import category of the queue item or the caller's choice. + Category string + // QueueID is the queue item this import belongs to (0: none yet). + QueueID int64 + // TargetPath is the queue item's forced scan path, kept for the later ARR + // notification. + TargetPath string + // SkipArr is set when the caller never wants the ARRs told. + SkipArr bool + // Metadata is the queue metadata JSON of the import (the media ids a + // Stremio-originated import stores, so the record this node later publishes + // carries them) and DownloadID its download id. Both are caller-supplied + // local data, never taken from the bundle; they may be empty. + Metadata string + DownloadID string + // DirFunc, when set and Dir is empty, chooses the local folder from the + // bundle's sanitised title (the caller applies its normal folder rules). + DirFunc func(title string) string +} + +// Request is one bundle to accept. +type Request struct { + Bundle *bundle.Bundle + Origin Origin + Target Target + // ExpectedNZB, when set, is the segment list of the NZB the user supplied + // (one slice per NZB file, in NZB order). The bundle's own list must equal + // it. + ExpectedNZB [][]string + // ExpectedNZBSizes, when set with ExpectedNZB, is the yEnc-ENCODED size of + // every segment of the user's NZB (same shape). Encoded sizes run a few + // percent above decoded ones, so they only bound the bundle's declared + // decoded sizes; they cannot prove them. + ExpectedNZBSizes [][]int64 +} + +// Result describes an accepted bundle. +type Result struct { + Title string + // Dir is the folder the files were written to. + Dir string + // Paths are the virtual paths of the files, in bundle order. + Paths []string + // Sizes are the file sizes, parallel to Paths. + Sizes []int64 + // TotalSize is the sum of the file sizes. + TotalSize int64 + // AlreadyImported is set when every file was already present locally with + // the same File ID (nothing was written). + AlreadyImported bool +} + +// Counters is a snapshot of the accept counters. +type Counters struct { + Accepted uint64 + AlreadyImported uint64 + Rejected map[Reason]uint64 + FullCheckPassed uint64 + FullCheckFailed uint64 + Announced uint64 + BadBundleReports uint64 +} + +type counters struct { + mu sync.Mutex + accepted uint64 + already uint64 + rejected map[Reason]uint64 + passed uint64 + failed uint64 + announce uint64 + reports uint64 +} + +func (c *counters) add(f func(*counters)) { + c.mu.Lock() + defer c.mu.Unlock() + f(c) +} + +func (c *counters) snapshot() Counters { + c.mu.Lock() + defer c.mu.Unlock() + rej := make(map[Reason]uint64, len(c.rejected)) + for k, v := range c.rejected { + rej[k] = v + } + return Counters{Accepted: c.accepted, AlreadyImported: c.already, Rejected: rej, FullCheckPassed: c.passed, + FullCheckFailed: c.failed, Announced: c.announce, BadBundleReports: c.reports} +} + +// YencSamples and YencRandom return the sampling bounds with defaults. +func (p Policy) YencSamples() int { + return defaultInt(p.MaxYencSamples, DefaultPolicy().MaxYencSamples) +} + +func (p Policy) YencRandom() int { + if p.YencRandomPerTable < 0 { + return 0 + } + if p.YencRandomPerTable == 0 { + return DefaultPolicy().YencRandomPerTable + } + return p.YencRandomPerTable +} diff --git a/internal/peernet/bundleimport/yenc_test.go b/internal/peernet/bundleimport/yenc_test.go new file mode 100644 index 000000000..c48b47d53 --- /dev/null +++ b/internal/peernet/bundleimport/yenc_test.go @@ -0,0 +1,92 @@ +package bundleimport + +import ( + "context" + "fmt" + "sync/atomic" + "testing" + + "github.com/javi11/nntppool/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAccept_SampledYencHeadersMustAgreeWithTheLayout(t *testing.T) { + lies := map[string]func(i int, m nntppool.YEncMeta) nntppool.YEncMeta{ + "part size": func(_ int, m nntppool.YEncMeta) nntppool.YEncMeta { m.PartSize++; return m }, + "part offset": func(_ int, m nntppool.YEncMeta) nntppool.YEncMeta { m.PartBegin += 7; return m }, + "file total": func(_ int, m nntppool.YEncMeta) nntppool.YEncMeta { m.FileSize *= 2; return m }, + } + for name, lie := range lies { + t.Run(name, func(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("yl", mkvSpec("movie.mkv")) + for i, id := range b.Pool { + beh := e.fp.BehaviorFor(id) + beh.YEnc = lie(i, beh.YEnc) + e.fp.SetBehavior(id, beh) + } + _, err := e.imp.Accept(context.Background(), e.req(b)) + require.Error(t, err) + assert.Equal(t, ReasonYencMismatch, ReasonOf(err)) + assert.Len(t, e.rep.got, 1, "the publisher is reported as having served a bad bundle") + e.assertNothingBehind() + }) + } +} + +func TestAccept_NoHeaderReaderMeansNoAccept(t *testing.T) { + e := newEnv(t) + e.imp.cfg.Headers = nil + _, err := e.imp.Accept(context.Background(), e.req(e.mkBundle("nh", mkvSpec("movie.mkv")))) + require.Error(t, err) + e.assertNothingBehind() +} + +type countingHeaders struct { + SegmentHeaders + n atomic.Int32 +} + +func (c *countingHeaders) SegmentHeader(ctx context.Context, id string) (YencHeader, error) { + c.n.Add(1) + return c.SegmentHeaders.SegmentHeader(ctx, id) +} + +func TestAccept_YencSamplesAreBounded(t *testing.T) { + e := newEnv(t) + files := []fileSpec{} + for i := 0; i < 10; i++ { + files = append(files, mkvSpec(fmt.Sprintf("m%d.mkv", i))) + } + b := e.mkBundle("cap", files...) + ch := &countingHeaders{SegmentHeaders: e.imp.cfg.Headers} + e.imp.cfg.Headers = ch + e.imp.cfg.Policy.MaxYencSamples = 6 + _, err := e.imp.Accept(context.Background(), e.req(b)) + require.NoError(t, err) + assert.EqualValues(t, 6, ch.n.Load()) +} + +func TestPlanSamples_ChoosesFirstLastAndUnpredictableMiddles(t *testing.T) { + tab := sampleTable{ids: make([]string, 200)} + seen := map[int]struct{}{} + for range 40 { + plan := planSamples([]sampleTable{tab}, 2, 24) + require.Len(t, plan, 4) + assert.Equal(t, 0, plan[0].pos) + assert.Equal(t, 199, plan[1].pos) + for _, s := range plan[2:] { + assert.Greater(t, s.pos, 0) + assert.Less(t, s.pos, 199) + seen[s.pos] = struct{}{} + } + } + assert.Greater(t, len(seen), 20, "middle positions must vary from call to call") + + many := make([]sampleTable, 50) + for i := range many { + many[i] = sampleTable{ids: make([]string, 10)} + } + assert.Len(t, planSamples(many, 2, 24), 24, "the total cap holds") +} diff --git a/internal/peernet/bundlex/buckets.go b/internal/peernet/bundlex/buckets.go new file mode 100644 index 000000000..29807bf52 --- /dev/null +++ b/internal/peernet/bundlex/buckets.go @@ -0,0 +1,96 @@ +package bundlex + +import ( + "net/netip" + "sync" + "time" +) + +// bucket is a token bucket measured in arbitrary units. +type bucket struct { + tokens float64 + last time.Time +} + +// buckets is a bounded set of token buckets keyed by string. +type buckets struct { + rate, burst float64 // units per second, units + max int + + mu sync.Mutex + m map[string]*bucket +} + +func newBuckets(perHour, burst, max int) *buckets { + return &buckets{rate: float64(perHour) / 3600, burst: float64(burst), max: max, m: map[string]*bucket{}} +} + +// take spends n units of key's bucket at now. +func (b *buckets) take(key string, n float64, now time.Time) bool { + b.mu.Lock() + defer b.mu.Unlock() + e := b.m[key] + if e == nil { + if len(b.m) >= b.max { + oldest, at := "", time.Time{} + for k, v := range b.m { // full: forget the least recently used key + if oldest == "" || v.last.Before(at) { + oldest, at = k, v.last + } + } + delete(b.m, oldest) + } + e = &bucket{tokens: b.burst, last: now} + b.m[key] = e + } + if d := now.Sub(e.last).Seconds(); d > 0 { + e.tokens = min(b.burst, e.tokens+d*b.rate) + } + e.last = now + if e.tokens < n { + return false + } + e.tokens -= n + return true +} + +// counter caps concurrent holders per key. +type counter struct { + mu sync.Mutex + m map[string]int +} + +func newCounter() *counter { return &counter{m: map[string]int{}} } + +func (c *counter) enter(key string, max int) bool { + c.mu.Lock() + defer c.mu.Unlock() + if c.m[key] >= max { + return false + } + c.m[key]++ + return true +} + +func (c *counter) leave(key string) { + c.mu.Lock() + defer c.mu.Unlock() + if c.m[key] <= 1 { + delete(c.m, key) + } else { + c.m[key]-- + } +} + +// sharedIP reports whether ip may be shared by honest nodes (private, +// loopback, link-local, unspecified): such an address is never limited or +// struck as a whole. Unparsable addresses count as shared too. +func sharedIP(ip string) bool { + a, err := netip.ParseAddr(ip) + if err != nil { + return true + } + a = a.Unmap() + return a.IsPrivate() || a.IsLoopback() || a.IsLinkLocalUnicast() || a.IsLinkLocalMulticast() || + a.IsUnspecified() || a.IsMulticast() +} diff --git a/internal/peernet/bundlex/build_bench_test.go b/internal/peernet/bundlex/build_bench_test.go new file mode 100644 index 000000000..825dd6d72 --- /dev/null +++ b/internal/peernet/bundlex/build_bench_test.go @@ -0,0 +1,34 @@ +package bundlex + +import ( + "context" + "fmt" + "testing" + + "github.com/kipsilabs/altmount/internal/database" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/bundle" +) + +// BenchmarkBuild75kSegments measures a rebuild of a realistic 50 GB release +// (75,000 segments, one file plus an unimported par2 set) from resolved metadata. +func BenchmarkBuild75kSegments(b *testing.B) { + video := make([]string, 75000) + for i := range video { + video[i] = fmt.Sprintf("part%d-%05dof75000.%x@news.example.net", i, i, uint64(i)*2654435761) + } + lib := &fakeLib{scheme: database.PeernetSchemeNZB, paths: []string{"p"}, + files: map[string]*metapb.FileMetadata{"p": plainFile("big.mkv", video)}, nzb: [][]string{video}} + rel := releaseHexOfNZB(lib.nzb) + bl := NewBuilder(BuilderConfig{Source: lib, Library: lib, Patches: NoPatches{}}) + b.ResetTimer() + for range b.N { + bl.Invalidate(rel) + got, err := bl.Build(context.Background(), rel, "Big.Release", posted) + if err != nil { + b.Fatal(err) + } + b.ReportMetric(float64(len(got.Wire))/1e6, "wireMB") + } + _ = bundle.MaxCompressedSize +} diff --git a/internal/peernet/bundlex/builder.go b/internal/peernet/bundlex/builder.go new file mode 100644 index 000000000..2781b16b1 --- /dev/null +++ b/internal/peernet/bundlex/builder.go @@ -0,0 +1,444 @@ +package bundlex + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "sync" + "time" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/holes" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/nzbname" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/contentid" + "github.com/kipsilabs/altmount/internal/peernet/records" +) + +// Reasons a release cannot be shared as a bundle (beyond the bundle package's +// own: rclone_encrypted, unsupported_encryption, password_not_from_nzb, +// healed_by_local_patch). +const ( + ReasonKnownHoles = "known_holes" + ReasonPatchUnknown = "patch_status_unknown" + ReasonNoNZBList = "no_nzb_segment_list" + ReasonReleaseMismatch = "release_id_mismatch" + ReasonInvalidMetadata = "invalid_metadata" + ReasonTooLarge = "too_large" + ReasonNoPostDate = "no_post_date" + ReasonNoFiles = "no_files" + ReasonSelfCheckFailed = "self_check_failed" + maxBuiltCacheBytes = 32 << 20 + builtTTL = 10 * time.Minute + defaultBuildWait = 30 * time.Second + maxReasonCounters = 64 + // serveBuildsPerHour and serveBuildBurst bound cache-missing builds for + // peer requests, node-wide. + serveBuildsPerHour = 360 + serveBuildBurst = 3 + negativeTTL = time.Minute + maxNegative = 256 +) + +// NotShareableError says a release cannot be shared as a bundle, for a +// permanent reason (the same input never succeeds). +type NotShareableError struct{ Reason string } + +func (e *NotShareableError) Error() string { return "bundlex: release not shareable: " + e.Reason } + +// ErrBusy is returned when the build slot did not become free in time. +var ErrBusy = errors.New("bundlex: another bundle is being built") + +// Built is a packed bundle. It is immutable; callers must not modify Wire. +type Built struct { + ReleaseID string + ID bundle.BundleID + Scheme bundle.Scheme + Wire []byte +} + +// BuilderConfig wires a Builder. +type BuilderConfig struct { + Source BuildSource + Library Library + Patches PatchIndex // nil: patch status unknown, nothing is shareable + Now func() time.Time + // BuildWait bounds the wait for the build slot (default 30 s). + BuildWait time.Duration +} + +// Builder turns a finished local import into a packed bundle. Builds are +// deterministic for the same metadata, one at a time, and the most recent +// results are kept in a small memory cache (nothing is written to disk). +type Builder struct { + cfg BuilderConfig + slot chan struct{} + // serveSlot and rate keep builds for peer requests apart from this node's + // own publishing builds, and bound them node-wide whoever asks. + serveSlot chan struct{} + rate *buckets + neg map[string]negEntry + + mu sync.Mutex + cache map[string]cacheEntry + byBundle map[bundle.BundleID]string + bytes int + notShare map[string]int64 + built int64 + hits int64 + failures int64 +} + +type negEntry struct { + err error + until time.Time +} + +type cacheEntry struct { + b *Built + at time.Time +} + +// NewBuilder returns a Builder. +func NewBuilder(cfg BuilderConfig) *Builder { + if cfg.Now == nil { + cfg.Now = time.Now + } + if cfg.BuildWait <= 0 { + cfg.BuildWait = defaultBuildWait + } + return &Builder{cfg: cfg, slot: make(chan struct{}, 1), serveSlot: make(chan struct{}, 1), + rate: newBuckets(serveBuildsPerHour, serveBuildBurst, 1), neg: map[string]negEntry{}, cache: map[string]cacheEntry{}, + byBundle: map[bundle.BundleID]string{}, notShare: map[string]int64{}} +} + +// BuilderStatus counts what the builder did since start. +type BuilderStatus struct { + Built, CacheHits, BuildFailures int64 + Cached, CachedBytes int + NotShareable map[string]int64 +} + +// Status returns the counters. +func (b *Builder) Status() BuilderStatus { + b.mu.Lock() + defer b.mu.Unlock() + ns := make(map[string]int64, len(b.notShare)) + for k, v := range b.notShare { + ns[k] = v + } + return BuilderStatus{Built: b.built, CacheHits: b.hits, BuildFailures: b.failures, + Cached: len(b.cache), CachedBytes: b.bytes, NotShareable: ns} +} + +// Invalidate forgets the cached bundle of a release. +func (b *Builder) Invalidate(releaseHex string) { + b.mu.Lock() + defer b.mu.Unlock() + b.drop(releaseHex) +} + +func (b *Builder) drop(rel string) { + if e, ok := b.cache[rel]; ok { + b.bytes -= len(e.b.Wire) + delete(b.byBundle, e.b.ID) + delete(b.cache, rel) + } +} + +func (b *Builder) cached(rel string) (*Built, bool) { + b.mu.Lock() + defer b.mu.Unlock() + e, ok := b.cache[rel] + if !ok { + return nil, false + } + if b.cfg.Now().Sub(e.at) > builtTTL { + b.drop(rel) + return nil, false + } + b.hits++ + return e.b, true +} + +func (b *Builder) store(rel string, built *Built) { + b.mu.Lock() + defer b.mu.Unlock() + b.drop(rel) + if len(built.Wire) > maxBuiltCacheBytes { + return // one wire larger than the whole cap is never cached + } + for b.bytes+len(built.Wire) > maxBuiltCacheBytes && len(b.cache) > 0 { + oldest, at := "", time.Time{} + for k, e := range b.cache { + if oldest == "" || e.at.Before(at) { + oldest, at = k, e.at + } + } + b.drop(oldest) + } + b.cache[rel] = cacheEntry{b: built, at: b.cfg.Now()} + b.byBundle[built.ID] = rel + b.bytes += len(built.Wire) +} + +func (b *Builder) count(reason string) { + b.mu.Lock() + defer b.mu.Unlock() + if _, ok := b.notShare[reason]; !ok && len(b.notShare) >= maxReasonCounters { + reason = "other" + } + b.notShare[reason]++ +} + +func (b *Builder) acquire(ctx context.Context) error { + t := time.NewTimer(b.cfg.BuildWait) + defer t.Stop() + select { + case b.slot <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return ErrBusy + } +} + +// Build returns the packed bundle of the release releaseHex. title is the +// release title and posted its post date, as the release record carries them. +// A *NotShareableError is permanent (and counted); any other error is +// transient. +func (b *Builder) Build(ctx context.Context, releaseHex, title string, posted time.Time) (*Built, error) { + if c, ok := b.cached(releaseHex); ok { + return c, nil + } + if err := b.acquire(ctx); err != nil { + return nil, err + } + defer func() { <-b.slot }() + if c, ok := b.cached(releaseHex); ok { // built while this call waited + return c, nil + } + return b.buildAndStore(ctx, releaseHex, title, posted) +} + +func (b *Builder) buildAndStore(ctx context.Context, releaseHex, title string, posted time.Time) (*Built, error) { + built, err := b.build(ctx, releaseHex, title, posted) + var ns *NotShareableError + switch { + case errors.As(err, &ns): + b.count(ns.Reason) + case err != nil: + b.mu.Lock() + b.failures++ + b.mu.Unlock() + default: + b.mu.Lock() + b.built++ + b.mu.Unlock() + b.store(releaseHex, built) + } + return built, err +} + +// BuildForServe is Build for a request from a peer: it has its own build slot +// (so a stream of peer requests can never starve this node's own publishing +// builds), a node-wide build rate independent of who asks, and remembers an +// unbuildable release for a short while. +func (b *Builder) BuildForServe(ctx context.Context, releaseHex, title string, posted time.Time) (*Built, error) { + if c, ok := b.cached(releaseHex); ok { + return c, nil + } + if err := b.negative(releaseHex); err != nil { + return nil, err + } + if !b.rate.take("all", 1, b.cfg.Now()) { + return nil, ErrBusy + } + t := time.NewTimer(b.cfg.BuildWait) + defer t.Stop() + select { + case b.serveSlot <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + case <-t.C: + return nil, ErrBusy + } + defer func() { <-b.serveSlot }() + if c, ok := b.cached(releaseHex); ok { + return c, nil + } + built, err := b.buildAndStore(ctx, releaseHex, title, posted) + var ns *NotShareableError + if errors.As(err, &ns) { + b.rememberNegative(releaseHex, err) + } + return built, err +} + +// negative returns the remembered error of a release that was recently found +// not shareable. +func (b *Builder) negative(rel string) error { + b.mu.Lock() + defer b.mu.Unlock() + n, ok := b.neg[rel] + if !ok { + return nil + } + if b.cfg.Now().After(n.until) { + delete(b.neg, rel) + return nil + } + return n.err +} + +func (b *Builder) rememberNegative(rel string, err error) { + b.mu.Lock() + defer b.mu.Unlock() + if len(b.neg) >= maxNegative { + oldest, at := "", time.Time{} + for k, n := range b.neg { + if oldest == "" || n.until.Before(at) { + oldest, at = k, n.until + } + } + delete(b.neg, oldest) + } + b.neg[rel] = negEntry{err: err, until: b.cfg.Now().Add(negativeTTL)} +} + +func notShareable(reason string) error { return &NotShareableError{Reason: reason} } + +func (b *Builder) build(ctx context.Context, releaseHex, title string, posted time.Time) (*Built, error) { + rid, err := contentid.ParseID(releaseHex) + if err != nil { + return nil, fmt.Errorf("bundlex: release id: %w", err) + } + if b.cfg.Patches == nil { + return nil, notShareable(ReasonPatchUnknown) + } + storedScheme, paths, err := b.cfg.Library.ReleaseFiles(ctx, releaseHex) + if err != nil { + return nil, fmt.Errorf("bundlex: list release files: %w", err) + } + switch { + case len(paths) == 0: + return nil, notShareable(ReasonNoFiles) + case len(paths) > bundle.MaxFiles: + return nil, notShareable(ReasonTooLarge) + case posted.Unix() < 1: + return nil, notShareable(ReasonNoPostDate) + } + scheme := bundle.SchemeFiles + if storedScheme != database.PeernetSchemeFiles { + scheme = bundle.SchemeNZB + } + in := bundle.ReleaseInput{Scheme: scheme, Title: records.CleanTitle(nzbname.ReleaseName(title, false)), PostedAt: posted} + for _, p := range paths { + if err := ctx.Err(); err != nil { + return nil, err + } + m, err := b.cfg.Source.ReadFile(p) + if err != nil { + return nil, fmt.Errorf("bundlex: read file metadata: %w", err) + } + if m == nil { + return nil, fmt.Errorf("bundlex: file metadata vanished") + } + if len(m.KnownHoles) > 0 { + return nil, notShareable(ReasonKnownHoles) + } + fi := bundle.FileInput{Meta: m, PatchedLocally: b.anyPatched(m)} + pw, fromNZB := b.cfg.Source.NZBPassword(p, m) + fi.PasswordFromNZB = fromNZB + if fromNZB && in.NZBPassword == "" { + in.NZBPassword = pw + } + in.Files = append(in.Files, fi) + } + if scheme == bundle.SchemeNZB { + lists, err := b.cfg.Source.NZBSegmentIDs(paths[0]) + if err != nil { + return nil, fmt.Errorf("bundlex: read nzb segment list: %w", err) + } + if len(lists) == 0 { + return nil, notShareable(ReasonNoNZBList) + } + in.NZBFiles = lists + } + bd, err := bundle.FromLocal(in) + if err != nil { + var nse *bundle.NotShareableError + switch { + case errors.As(err, &nse): + return nil, notShareable(string(nse.Reason)) + case errors.Is(err, bundle.ErrLimit), errors.Is(err, bundle.ErrTooLarge): + return nil, notShareable(ReasonTooLarge) + } + return nil, notShareable(ReasonInvalidMetadata) + } + if bd.ReleaseID != rid { + return nil, notShareable(ReasonReleaseMismatch) + } + wire, id, err := bundle.Pack(bd) + if err != nil { + if errors.Is(err, bundle.ErrTooLarge) { + return nil, notShareable(ReasonTooLarge) + } + return nil, notShareable(ReasonInvalidMetadata) + } + if _, err := bundle.Verify(wire, id, rid); err != nil { + return nil, notShareable(ReasonSelfCheckFailed) + } + return &Built{ReleaseID: releaseHex, ID: id, Scheme: scheme, Wire: wire}, nil +} + +// anyPatched reports whether a local PAR2 patch covers any article of the +// file's main or outer segments. +func (b *Builder) anyPatched(m *metapb.FileMetadata) bool { + has := func(segs []*metapb.SegmentData) bool { + for _, s := range segs { + if s != nil && !holes.IsPlaceholderID(s.Id) && b.cfg.Patches.Has(trimIDBrackets(s.Id)) { + return true + } + } + return false + } + if has(m.SegmentData) { + return true + } + for _, ns := range m.NestedSources { + if ns != nil && has(ns.Segments) { + return true + } + } + for _, ns := range m.SharedOuterSources { + if ns != nil && has(ns.Segments) { + return true + } + } + return false +} + +func trimIDBrackets(id string) string { + if len(id) > 1 && id[0] == '<' && id[len(id)-1] == '>' { + return id[1 : len(id)-1] + } + return id +} + +// LookupCached returns the cached bundle with this ID, if any. +func (b *Builder) LookupCached(id bundle.BundleID) (*Built, bool) { + b.mu.Lock() + rel, ok := b.byBundle[id] + b.mu.Unlock() + if !ok { + return nil, false + } + return b.cached(rel) +} + +// HexID renders a bundle ID. +func HexID(id bundle.BundleID) string { return hex.EncodeToString(id[:]) } diff --git a/internal/peernet/bundlex/builder_test.go b/internal/peernet/bundlex/builder_test.go new file mode 100644 index 000000000..4bc571a6d --- /dev/null +++ b/internal/peernet/bundlex/builder_test.go @@ -0,0 +1,219 @@ +package bundlex + +import ( + "bytes" + "context" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/database" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +func notShareableReason(t *testing.T, err error) string { + t.Helper() + var ns *NotShareableError + if !errors.As(err, &ns) { + t.Fatalf("want NotShareableError, got %v", err) + } + return ns.Reason +} + +func TestBuildNZBSchemeVerifiesAgainstItsReleaseID(t *testing.T) { + lib, rel := nzbRelease() + b := newTestBuilder(t, lib, NoPatches{}) + got, err := b.Build(context.Background(), rel, "Some.Movie.2020.1080p", posted) + if err != nil { + t.Fatal(err) + } + rid, _ := contentid.ParseID(rel) + bd, err := bundle.Verify(got.Wire, got.ID, rid) + if err != nil { + t.Fatalf("a receiver would reject the bundle: %v", err) + } + if bd.Scheme != bundle.SchemeNZB || len(bd.Pool) != 23 { + t.Fatalf("scheme %v pool %d (the whole NZB list must travel)", bd.Scheme, len(bd.Pool)) + } +} + +func TestBuildIsDeterministic(t *testing.T) { + lib, rel := nzbRelease() + a, err := newTestBuilder(t, lib, NoPatches{}).Build(context.Background(), rel, "T", posted) + if err != nil { + t.Fatal(err) + } + b, err := newTestBuilder(t, lib, NoPatches{}).Build(context.Background(), rel, "T", posted) + if err != nil { + t.Fatal(err) + } + if a.ID != b.ID || !bytes.Equal(a.Wire, b.Wire) { + t.Fatal("two builds of the same metadata differ") + } +} + +func TestBuildFilesSchemeNeverFallsBackFromNZB(t *testing.T) { + lib, rel := nzbRelease() + lib.nzb = nil // the store is gone + _, err := newTestBuilder(t, lib, NoPatches{}).Build(context.Background(), rel, "T", posted) + if r := notShareableReason(t, err); r != ReasonNoNZBList { + t.Fatalf("reason = %s", r) + } + // A files-scheme release is built from its files alone. + f := plainFile("a.mkv", ids("a", 5)) + fid, _ := contentid.FileID(f) + frid, _ := contentid.ReleaseIDFromFiles([]contentid.ID{fid}) + lib2 := &fakeLib{scheme: database.PeernetSchemeFiles, paths: []string{"p"}, files: map[string]*metapb.FileMetadata{"p": f}} + got, err := newTestBuilder(t, lib2, NoPatches{}).Build(context.Background(), frid.String(), "T", posted) + if err != nil || got.Scheme != bundle.SchemeFiles { + t.Fatalf("files scheme: %v %v", got, err) + } +} + +func TestBuildStaleLibraryIsAReleaseIDMismatch(t *testing.T) { + lib, rel := nzbRelease() + lib.nzb = append(lib.nzb, ids("extra", 1)) // the NZB list no longer hashes to the indexed ID + _, err := newTestBuilder(t, lib, NoPatches{}).Build(context.Background(), rel, "T", posted) + if r := notShareableReason(t, err); r != ReasonReleaseMismatch { + t.Fatalf("reason = %s", r) + } +} + +func TestNotShareableReasonsAreCounted(t *testing.T) { + cases := map[string]func(*fakeLib) PatchIndex{ + "rclone_encrypted": func(l *fakeLib) PatchIndex { + l.files["movies/R/movie.mkv"].Encryption = metapb.Encryption_RCLONE + return NoPatches{} + }, + "healed_by_local_patch": func(l *fakeLib) PatchIndex { return patches{"vid-000003@news.test": true} }, + "known_holes": func(l *fakeLib) PatchIndex { + l.files["movies/R/movie.mkv"].KnownHoles = []*metapb.HoleRun{{StartSegment: 1, Count: 1}} + return NoPatches{} + }, + "password_not_from_nzb": func(l *fakeLib) PatchIndex { + m := l.files["movies/R/movie.mkv"] + m.Encryption, m.AesKey, m.AesIv = metapb.Encryption_AES, bytes.Repeat([]byte{1}, 32), bytes.Repeat([]byte{2}, 16) + return NoPatches{} + }, + "patch_status_unknown": func(l *fakeLib) PatchIndex { return nil }, + } + for want, mut := range cases { + lib, rel := nzbRelease() + b := newTestBuilder(t, lib, mut(lib)) + _, err := b.Build(context.Background(), rel, "T", posted) + if r := notShareableReason(t, err); r != want { + t.Errorf("%s: reason = %s", want, r) + } + if b.Status().NotShareable[want] != 1 { + t.Errorf("%s: not counted: %v", want, b.Status().NotShareable) + } + } +} + +func TestPasswordTravelsOnlyWhenFromTheNZB(t *testing.T) { + enc := func() (*fakeLib, string) { + lib, rel := nzbRelease() + m := lib.files["movies/R/movie.mkv"] + m.Encryption, m.AesKey, m.AesIv = metapb.Encryption_AES, bytes.Repeat([]byte{0x41}, 32), bytes.Repeat([]byte{0x42}, 16) + m.FileSize += 0 + return lib, rel + } + // Not from the NZB (the metadata adapter's answer): not shareable. + lib, rel := enc() + if _, err := newTestBuilder(t, lib, NoPatches{}).Build(context.Background(), rel, "T", posted); err == nil { + t.Fatal("key material shared without a proven NZB password") + } + // From the NZB: the key and the password travel. + lib, rel = enc() + lib.pwNZB = map[string]string{"movies/R/movie.mkv": "NZB-PW-MARK"} + got, err := newTestBuilder(t, lib, NoPatches{}).Build(context.Background(), rel, "T", posted) + if err != nil { + // A segment table of unencrypted size is not an AES-padded one; the + // structural validator may refuse the synthetic file, which is fine + // for this test only if the reason is not the password rule. + if r := notShareableReason(t, err); r == "password_not_from_nzb" { + t.Fatalf("the password rule fired for a from-NZB password") + } + return + } + raw, _ := bundle.Decompress(got.Wire) + if !bytes.Contains(raw, []byte("NZB-PW-MARK")) { + t.Fatal("NZB password missing from a from-NZB bundle") + } +} + +func TestBundleBytesContainNoLocalData(t *testing.T) { + lib, rel := nzbRelease() + m := lib.files["movies/R/movie.mkv"] + m.Password, m.Salt = "RCLONE-PASSWORD-MARK", "RCLONE-SALT-MARK" + m.KnownHoles = nil + got, err := newTestBuilder(t, lib, NoPatches{}).Build(context.Background(), rel, "Some.Movie.2020", posted) + if err != nil { + t.Fatal(err) + } + raw, err := bundle.Decompress(got.Wire) + if err != nil { + t.Fatal(err) + } + for _, mark := range privateMarkers { + if bytes.Contains(raw, []byte(mark)) || bytes.Contains(got.Wire, []byte(mark)) { + t.Errorf("bundle contains %q", mark) + } + } + if strings.Contains(string(raw), "movies/R") { + t.Error("bundle contains a virtual path") + } +} + +func TestBuildConcurrencyIsOneAndResultCached(t *testing.T) { + lib, rel := nzbRelease() + lib.slowRead = 50 * time.Millisecond + b := newTestBuilder(t, lib, NoPatches{}) + var wg sync.WaitGroup + for range 5 { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := b.Build(context.Background(), rel, "T", posted); err != nil { + t.Error(err) + } + }() + } + wg.Wait() + if lib.readCount() != 1 { + t.Fatalf("metadata read %d times, want one build", lib.readCount()) + } + if st := b.Status(); st.Built != 1 || st.CacheHits != 4 || st.Cached != 1 { + t.Fatalf("status %+v", st) + } + b.Invalidate(rel) + if _, err := b.Build(context.Background(), rel, "T", posted); err != nil || lib.readCount() != 2 { + t.Fatalf("after invalidate: %v reads=%d", err, lib.readCount()) + } +} + +func TestBuildBusyAndTransientErrors(t *testing.T) { + lib, rel := nzbRelease() + lib.slowRead = 300 * time.Millisecond + b := NewBuilder(BuilderConfig{Source: lib, Library: lib, Patches: NoPatches{}, BuildWait: 20 * time.Millisecond}) + go func() { _, _ = b.Build(context.Background(), rel, "T", posted) }() + time.Sleep(50 * time.Millisecond) + if _, err := b.Build(context.Background(), "ab"+rel[2:], "T", posted); !errors.Is(err, ErrBusy) { + t.Fatalf("want ErrBusy, got %v", err) + } + time.Sleep(400 * time.Millisecond) + lib.setReadErr(errors.New("disk")) + b.Invalidate(rel) + _, err := b.Build(context.Background(), rel, "T", posted) + var ns *NotShareableError + if err == nil || errors.As(err, &ns) { + t.Fatalf("a read failure must be transient, got %v", err) + } + if b.Status().BuildFailures != 1 { + t.Fatal("failure not counted") + } +} diff --git a/internal/peernet/bundlex/dos_test.go b/internal/peernet/bundlex/dos_test.go new file mode 100644 index 000000000..c8237105b --- /dev/null +++ b/internal/peernet/bundlex/dos_test.go @@ -0,0 +1,96 @@ +package bundlex + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/peernet/identity" + "github.com/kipsilabs/altmount/internal/peernet/netid" +) + +func TestStrikeTablesEvictTheOldestExpiryNotARandomEntry(t *testing.T) { + e := newFetchEnv(t, FetchLimits{MaxTracked: 3}) + now := time.Now() + mk := func() identity.NodeID { k, _ := identity.GenerateKey(); return k.NodeID() } + a, b, c, d := mk(), mk(), mk(), mk() + e.f.mu.Lock() + e.f.strikes[strikeKey(nwA, a)] = &strike{n: 5, until: now.Add(2 * time.Hour)} + e.f.strikes[strikeKey(nwA, b)] = &strike{n: 1, until: now.Add(time.Minute)} // expires first + e.f.strikes[strikeKey(nwA, c)] = &strike{n: 5, until: now.Add(3 * time.Hour)} + e.f.mu.Unlock() + e.f.fail(nwA, d, failTransport, true) + e.f.mu.Lock() + defer e.f.mu.Unlock() + if _, kept := e.f.strikes[strikeKey(nwA, b)]; kept { + t.Fatal("the entry that expires first must be the one evicted") + } + for _, p := range []identity.NodeID{a, c, d} { + if _, ok := e.f.strikes[strikeKey(nwA, p)]; !ok { + t.Fatalf("a longer-lived strike was evicted") + } + } +} + +func TestBuilderDoesNotCacheAWireLargerThanItsCap(t *testing.T) { + lib, _ := nzbRelease() + b := newTestBuilder(t, lib, NoPatches{}) + b.store("rel", &Built{Wire: make([]byte, maxBuiltCacheBytes+1)}) + if st := b.Status(); st.CachedBytes > maxBuiltCacheBytes || st.Cached != 0 { + t.Fatalf("cache holds %d items / %d bytes, cap %d", st.Cached, st.CachedBytes, maxBuiltCacheBytes) + } +} + +func TestServeBuildsNeverBlockPublishBuilds(t *testing.T) { + lib, rel := nzbRelease() + lib.slowRead = 300 * time.Millisecond + b := NewBuilder(BuilderConfig{Source: lib, Library: lib, Patches: NoPatches{}, BuildWait: 50 * time.Millisecond}) + started := make(chan struct{}) + go func() { + close(started) + _, _ = b.BuildForServe(context.Background(), rel, "T", posted) + }() + <-started + time.Sleep(50 * time.Millisecond) // the serve build now holds its slot and is reading + _, err := b.Build(context.Background(), rel, "T", posted) + if errors.Is(err, ErrBusy) { + t.Fatal("a publish build waited behind a serve build") + } +} + +func TestServeBuildsAreRateLimitedNodeWideAndUnbuildableIsRememberedBriefly(t *testing.T) { + lib, _ := nzbRelease() + b := newTestBuilder(t, lib, NoPatches{}) + busy := 0 + for i := 0; i < 20; i++ { + rel := netid.Open().String()[:62] + string(rune('a'+i/16)) + string(rune('a'+i%16)) // distinct valid hex-ish ids + _ = rel + id := make([]byte, 32) + id[0], id[1] = byte(i), byte(i>>8) + hexID := "" + for _, c := range id { + hexID += string("0123456789abcdef"[c>>4]) + string("0123456789abcdef"[c&15]) + } + if _, err := b.BuildForServe(context.Background(), hexID, "T", posted); errors.Is(err, ErrBusy) { + busy++ + } + } + if busy == 0 { + t.Fatal("20 distinct serve builds in a burst must hit the node-wide build limit") + } + + lib2, rel2 := nzbRelease() + b2 := newTestBuilder(t, lib2, nil) // patch status unknown: permanently unbuildable + _, err := b2.BuildForServe(context.Background(), rel2, "T", posted) + if err == nil { + t.Fatal("expected a not-shareable error") + } + reads := lib2.readCount() + if _, err2 := b2.BuildForServe(context.Background(), rel2, "T", posted); err2 == nil { + t.Fatal("expected the remembered error") + } + if lib2.readCount() != reads { + t.Fatal("the second attempt must be answered from the negative cache") + } +} diff --git a/internal/peernet/bundlex/fetch.go b/internal/peernet/bundlex/fetch.go new file mode 100644 index 000000000..d86fbee71 --- /dev/null +++ b/internal/peernet/bundlex/fetch.go @@ -0,0 +1,581 @@ +package bundlex + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "sort" + "sync" + "time" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/contentid" + "github.com/kipsilabs/altmount/internal/peernet/discovery" + "github.com/kipsilabs/altmount/internal/peernet/identity" + "github.com/kipsilabs/altmount/internal/peernet/netid" + "github.com/kipsilabs/altmount/internal/peernet/records" + "github.com/kipsilabs/altmount/internal/peernet/recsync" + "github.com/kipsilabs/altmount/internal/peernet/transport" +) + +// Errors of Fetch. +var ( + ErrNoOffers = errors.New("bundlex: no fetchable offer for this release") + ErrRateLimited = errors.New("bundlex: fetch rate limit reached") + ErrFetchBusy = errors.New("bundlex: too many bundle downloads in progress") + ErrAllFailed = errors.New("bundlex: no offered bundle could be fetched and verified") +) + +// Fetch failure reasons (counted in the status). +const ( + failRefused = "refused" + failStatus = "bad_status" + failTransport = "transport" + failTruncated = "truncated" + failTooLarge = "too_large" + failVerification = "verification_failed" + failScheme = "scheme_mismatch" + failMembership = "membership" +) + +// OfferStore finds stored release records that announce a bundle. +type OfferStore interface { + BundleOffers(ctx context.Context, networkIDs []string, releaseID string, limit int) ([]database.PeernetBundleOffer, error) + AttestationsFor(ctx context.Context, networkID, releaseID, fileID string, notBefore time.Time) ([]database.PeernetAttestation, error) +} + +// PeerBook knows the verified peers of a network. +type PeerBook interface { + // VerifiedPeer returns the peer with a verified address, or false. + VerifiedPeer(ctx context.Context, nw netid.NetworkID, node identity.NodeID) (recsync.Peer, bool, error) + // Known returns which of nodes are known verified peers of nw. + Known(ctx context.Context, nw netid.NetworkID, nodes []string) (map[string]bool, error) +} + +// ReceivingNetworks lists the networks this node receives from. +type ReceivingNetworks interface { + ReceivingNetworks() []recsync.NetworkInfo +} + +// FetchConfig wires a Fetcher. +type FetchConfig struct { + Self identity.NodeID + Store OfferStore + Peers PeerBook + Networks ReceivingNetworks + Dialer recsync.Dialer + Auth discovery.Authorizer + Limits FetchLimits + Now func() time.Time +} + +// Offer is one stored release record that announces a bundle for a release. +type Offer struct { + Network netid.NetworkID + Publisher identity.NodeID + BundleID bundle.BundleID + Scheme bundle.Scheme + CreatedAt time.Time + // Known: the publisher is a known verified peer. AttestedHealthy: it has a + // fresh healthy attestation for this release ID. + Known, AttestedHealthy bool + // Fetchable: the publisher is a verified peer with an address in that + // network. A record relayed from a publisher this node has no verified + // address for cannot be fetched: bundles come from the publisher itself. + Fetchable bool + // Own: the record is this node's own. + Own bool + // Corroborators is how many distinct network ranges (/24 for IPv4, /48 for + // IPv6) of known verified publishers announce this very Bundle ID (at + // least 1 for a known verified publisher, 0 otherwise). + Corroborators int +} + +// FetchSource says where a fetched bundle came from. +type FetchSource struct { + Network netid.NetworkID + Publisher identity.NodeID + BundleID bundle.BundleID + Scheme bundle.Scheme + // Corroborators is the offer's count of distinct network ranges announcing + // the same Bundle ID. + Corroborators int +} + +// Fetcher downloads and verifies bundles on demand. +type Fetcher struct { + cfg FetchConfig + lim FetchLimits + glob *buckets + pub *buckets + slot chan struct{} + + mu sync.Mutex + strikes map[string]*strike + ok int64 + failed map[string]int64 + refused int64 + + // fcStrikes are the full-check penalties (see ReportBadBundle), kept apart + // from the strikes of hostile bytes. + fcStrikes map[string]*fcStrike + fcReports int64 +} + +type fcStrike struct { + n int + until, last time.Time +} + +type strike struct { + n int + until time.Time +} + +// NewFetcher returns a Fetcher. +func NewFetcher(cfg FetchConfig) *Fetcher { + if cfg.Now == nil { + cfg.Now = time.Now + } + if cfg.Auth == nil { + cfg.Auth = discovery.AllowAll{} + } + l := cfg.Limits.withDefaults() + return &Fetcher{cfg: cfg, lim: l, + glob: newBuckets(l.GlobalPerHour, l.GlobalBurst, 1), pub: newBuckets(l.PerPublisherPerHour, l.PerPublisherBurst, l.MaxTracked), + slot: make(chan struct{}, l.MaxConcurrent), strikes: map[string]*strike{}, fcStrikes: map[string]*fcStrike{}, failed: map[string]int64{}} +} + +// FetchStatus counts downloads since start. +type FetchStatus struct { + OK int64 + Failed map[string]int64 + RateLimited int64 + BackedOff int + // FullCheckReports counts bundles reported as failing the local full check; + // FullCheckBackedOff the publishers skipped for that reason now. + FullCheckReports int64 + FullCheckBackedOff int +} + +// Status returns the counters. +func (f *Fetcher) Status() FetchStatus { + f.mu.Lock() + defer f.mu.Unlock() + st := FetchStatus{OK: f.ok, Failed: map[string]int64{}, RateLimited: f.refused, FullCheckReports: f.fcReports} + for k, v := range f.failed { + st.Failed[k] = v + } + now := f.cfg.Now() + for _, s := range f.strikes { + if s.until.After(now) { + st.BackedOff++ + } + } + for _, s := range f.fcStrikes { + if s.until.After(now) { + st.FullCheckBackedOff++ + } + } + return st +} + +func strikeKey(nw netid.NetworkID, p identity.NodeID) string { return nw.String() + "|" + p.String() } + +func (f *Fetcher) backedOff(nw netid.NetworkID, p identity.NodeID) bool { + f.mu.Lock() + defer f.mu.Unlock() + now := f.cfg.Now() + if s := f.strikes[strikeKey(nw, p)]; s != nil && s.until.After(now) { + return true + } + s := f.fcStrikes[strikeKey(nw, p)] + return s != nil && s.until.After(now) +} + +func (f *Fetcher) fail(nw netid.NetworkID, p identity.NodeID, reason string, hostile bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.failed[reason]++ + k := strikeKey(nw, p) + s := f.strikes[k] + if s == nil { + if len(f.strikes) >= f.lim.MaxTracked { + oldest, at := "", time.Time{} + for kk, st := range f.strikes { + if oldest == "" || st.until.Before(at) { + oldest, at = kk, st.until + } + } + delete(f.strikes, oldest) + } + s = &strike{} + f.strikes[k] = s + } + wait := f.lim.SoftBackoff + if hostile { + s.n++ + wait = f.lim.BackoffBase + for i := 1; i < s.n && wait < f.lim.BackoffMax; i++ { + wait *= 2 + } + wait = min(wait, f.lim.BackoffMax) + } + if until := f.cfg.Now().Add(wait); until.After(s.until) { + s.until = until + } +} + +// Offers lists the offers for a release across the networks this node +// receives from, best first: known publishers, then those with a fresh +// healthy attestation, then newest. At most MaxOffers are returned. +func (f *Fetcher) Offers(ctx context.Context, releaseHex string) ([]Offer, error) { + if _, err := contentid.ParseID(releaseHex); err != nil { + return nil, fmt.Errorf("bundlex: release id: %w", err) + } + nets := f.cfg.Networks.ReceivingNetworks() + if len(nets) == 0 { + return nil, nil + } + if len(nets) > database.PeernetMaxNetworksPerQuery { + nets = nets[:database.PeernetMaxNetworksPerQuery] + } + ids := make([]string, len(nets)) + byID := make(map[string]netid.NetworkID, len(nets)) + for i, n := range nets { + ids[i] = n.ID.String() + byID[ids[i]] = n.ID + } + rows, err := f.cfg.Store.BundleOffers(ctx, ids, releaseHex, f.lim.MaxOffers) + if err != nil { + return nil, err + } + known := map[string]map[string]bool{} + attesters := map[string]map[string]bool{} + var out []Offer + for _, r := range rows { + nw, ok := byID[r.NetworkID] + pub, perr := identity.ParseNodeID(r.PublisherNodeID) + raw, berr := hexTo32(r.BundleID) + if !ok || perr != nil || berr != nil { + continue + } + o := Offer{Network: nw, Publisher: pub, BundleID: raw, Scheme: bundle.Scheme(schemeOf(r.ReleaseScheme)), + CreatedAt: r.CreatedAt, Own: r.Source == database.PeernetRecordSourceLocal || pub.Equal(f.cfg.Self)} + if _, done := known[r.NetworkID]; !done { + nodes := make([]string, 0, len(rows)) + for _, rr := range rows { + if rr.NetworkID == r.NetworkID { + nodes = append(nodes, rr.PublisherNodeID) + } + } + k, err := f.cfg.Peers.Known(ctx, nw, nodes) + if err != nil { + return nil, err + } + known[r.NetworkID] = k + set := map[string]bool{} + atts, err := f.cfg.Store.AttestationsFor(ctx, r.NetworkID, releaseHex, "", f.cfg.Now().Add(-records.AttestationValidity)) + if err != nil { + return nil, err + } + for _, a := range atts { + if a.Healthy { + set[a.PublisherNodeID] = true + } + } + attesters[r.NetworkID] = set + } + o.Known = known[r.NetworkID][r.PublisherNodeID] + // A publisher's own attestation never raises the rank of its own offer. + for q := range attesters[r.NetworkID] { + if q != r.PublisherNodeID { + o.AttestedHealthy = true + break + } + } + if !o.Own { + _, ok, err := f.cfg.Peers.VerifiedPeer(ctx, nw, pub) + if err != nil { + return nil, err + } + o.Fetchable = ok + } + out = append(out, o) + } + if err := f.corroborate(ctx, out); err != nil { + return nil, err + } + sort.SliceStable(out, func(i, j int) bool { + a, b := out[i], out[j] + if a.Known != b.Known { + return a.Known + } + if a.AttestedHealthy != b.AttestedHealthy { + return a.AttestedHealthy + } + if a.Corroborators != b.Corroborators { + return a.Corroborators > b.Corroborators + } + return a.CreatedAt.After(b.CreatedAt) + }) + return out, nil +} + +// rangeOf returns the network range of a peer address: /24 for IPv4, /48 for +// IPv6, the host itself for a name. +func rangeOf(addr string) string { + host, _, err := net.SplitHostPort(addr) + if err != nil { + host = addr + } + ip := net.ParseIP(host) + switch { + case ip == nil: + return "host:" + host + case ip.To4() != nil: + return ip.Mask(net.CIDRMask(24, 32)).String() + "/24" + } + return ip.Mask(net.CIDRMask(48, 128)).String() + "/48" +} + +// corroborate sets, per offer, how many distinct network ranges of known +// verified publishers announce the same Bundle ID. This node's own records and +// publishers without a verified address do not count. +func (f *Fetcher) corroborate(ctx context.Context, offers []Offer) error { + ranges := map[bundle.BundleID]map[string]struct{}{} + for _, o := range offers { + if o.Own || !o.Known { + continue + } + peer, ok, err := f.cfg.Peers.VerifiedPeer(ctx, o.Network, o.Publisher) + if err != nil { + return err + } + if !ok { + continue + } + if ranges[o.BundleID] == nil { + ranges[o.BundleID] = map[string]struct{}{} + } + ranges[o.BundleID][rangeOf(peer.Address)] = struct{}{} + } + for i := range offers { + offers[i].Corroborators = len(ranges[offers[i].BundleID]) + } + return nil +} + +func schemeOf(stored int) uint8 { + if stored == database.PeernetSchemeFiles { + return uint8(bundle.SchemeFiles) + } + return uint8(bundle.SchemeNZB) +} + +func hexTo32(s string) (bundle.BundleID, error) { + var id bundle.BundleID + if len(s) != 64 || !lowerHex(s) { + return id, errors.New("bad id") + } + for i := 0; i < 32; i++ { + id[i] = unhex(s[2*i])<<4 | unhex(s[2*i+1]) + } + return id, nil +} + +func unhex(c byte) byte { + if c >= 'a' { + return c - 'a' + 10 + } + return c - '0' +} + +// Fetch downloads the bundle of release releaseHex from the best offers (at +// most MaxAttempts) and returns the verified bundle and where it came from. +// The Bundle ID and Release ID checked are those of the signed record, never +// anything in the response. A publisher whose bytes fail verification is +// backed off. Fetch never runs speculatively: only a local caller starts it. +func (f *Fetcher) Fetch(ctx context.Context, releaseHex string) (*bundle.Bundle, FetchSource, error) { + rid, err := contentid.ParseID(releaseHex) + if err != nil { + return nil, FetchSource{}, fmt.Errorf("bundlex: release id: %w", err) + } + offers, err := f.Offers(ctx, releaseHex) + if err != nil { + return nil, FetchSource{}, err + } + var cands []Offer + for _, o := range offers { + if o.Fetchable && !f.backedOff(o.Network, o.Publisher) { + cands = append(cands, o) + } + } + if len(cands) == 0 { + return nil, FetchSource{}, ErrNoOffers + } + select { + case f.slot <- struct{}{}: + defer func() { <-f.slot }() + default: + return nil, FetchSource{}, ErrFetchBusy + } + var errs []error + attempts := 0 + for _, o := range cands { + if attempts >= f.lim.MaxAttempts || ctx.Err() != nil { + break + } + now := f.cfg.Now() + if !f.glob.take("all", 1, now) { + f.mu.Lock() + f.refused++ + f.mu.Unlock() + if attempts == 0 { + return nil, FetchSource{}, ErrRateLimited + } + break + } + if !f.pub.take(strikeKey(o.Network, o.Publisher), 1, now) { + continue + } + attempts++ + b, err := f.attempt(ctx, o, rid) + if err == nil { + f.mu.Lock() + f.ok++ + f.mu.Unlock() + return b, FetchSource{Network: o.Network, Publisher: o.Publisher, BundleID: o.BundleID, Scheme: o.Scheme, Corroborators: o.Corroborators}, nil + } + errs = append(errs, err) + } + if ctx.Err() != nil { + return nil, FetchSource{}, ctx.Err() + } + if len(errs) == 0 { + return nil, FetchSource{}, ErrNoOffers + } + return nil, FetchSource{}, fmt.Errorf("%w: %w", ErrAllFailed, errors.Join(errs...)) +} + +type fetchError struct{ reason string } + +func (e *fetchError) Error() string { return "bundlex: fetch failed: " + e.reason } + +func (f *Fetcher) attempt(ctx context.Context, o Offer, rid contentid.ID) (*bundle.Bundle, error) { + nets := f.cfg.Networks.ReceivingNetworks() + var info recsync.NetworkInfo + for _, n := range nets { + if n.ID == o.Network { + info = n + } + } + peer, ok, err := f.cfg.Peers.VerifiedPeer(ctx, o.Network, o.Publisher) + if err != nil || !ok || info.ID.IsZero() { + f.fail(o.Network, o.Publisher, failTransport, false) + return nil, &fetchError{failTransport} + } + hdr := http.Header{} + if err := f.cfg.Auth.DecorateOutbound(ctx, o.Network, o.Publisher, hdr); err != nil { + f.fail(o.Network, o.Publisher, failMembership, false) + return nil, &fetchError{failMembership} + } + resp, err := f.cfg.Dialer.Do(ctx, peer.Address, peer.NodeID, transport.Request{ + Method: http.MethodGet, Path: PathBundle, Header: hdr, + Query: url.Values{queryNet: {o.Network.String()}, queryBundle: {o.BundleID.String()}}, + }, transport.CallOptions{AllowPrivate: info.AllowPrivate, TrustedAddress: peer.Trusted, + MaxResponseBytes: bundle.MaxCompressedSize, Timeout: f.lim.Timeout}) + if err != nil { + reason, hostile := failTransport, false + if errors.Is(err, transport.ErrBodyTooLarge) { + reason, hostile = failTooLarge, true + } + f.fail(o.Network, o.Publisher, reason, hostile) + return nil, &fetchError{reason} + } + if err := f.cfg.Auth.VerifyResponse(ctx, o.Network, o.Publisher, resp.StatusCode, resp.Header); err != nil { + _ = resp.Body.Close() + f.fail(o.Network, o.Publisher, failMembership, false) + return nil, &fetchError{failMembership} + } + if resp.StatusCode != http.StatusOK { + _ = resp.Body.Close() + reason := failStatus + if resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests { + reason = failRefused + } + f.fail(o.Network, o.Publisher, reason, false) + return nil, &fetchError{reason} + } + wire, err := resp.ReadAll() + if err != nil { + reason, hostile := failTruncated, false + if errors.Is(err, transport.ErrBodyTooLarge) { + reason, hostile = failTooLarge, true + } + f.fail(o.Network, o.Publisher, reason, hostile) + return nil, &fetchError{reason} + } + b, err := bundle.Verify(wire, o.BundleID, rid) + if err != nil { + f.fail(o.Network, o.Publisher, failVerification, true) + return nil, &fetchError{failVerification} + } + if b.Scheme != o.Scheme { + f.fail(o.Network, o.Publisher, failScheme, true) + return nil, &fetchError{failScheme} + } + return b, nil +} + +// ReportBadBundle records that a bundle from publisher p passed hash +// verification but its release then failed the local full health check. The +// publisher is skipped for bundle fetches only: 30 minutes, doubling per +// report, capped at 6 hours, and the doubling decays after a day without a +// report. It never touches record sync or stored records. The penalty is mild +// because the same failure is also produced by this node's own providers +// lacking articles the publisher's had; a later verified bundle from the same +// publisher clears it (ReportVerifiedBundle). +func (f *Fetcher) ReportBadBundle(nw netid.NetworkID, p identity.NodeID) { + f.mu.Lock() + defer f.mu.Unlock() + f.fcReports++ + now := f.cfg.Now() + k := strikeKey(nw, p) + s := f.fcStrikes[k] + if s == nil { + if len(f.fcStrikes) >= f.lim.MaxTracked { + oldest, at := "", time.Time{} + for kk, st := range f.fcStrikes { + if oldest == "" || st.until.Before(at) { + oldest, at = kk, st.until + } + } + delete(f.fcStrikes, oldest) + } + s = &fcStrike{} + f.fcStrikes[k] = s + } + if !s.last.IsZero() && now.Sub(s.last) > f.lim.FullCheckDecay { + s.n = 0 + } + s.n++ + s.last = now + wait := f.lim.FullCheckBackoffBase + for i := 1; i < s.n && wait < f.lim.FullCheckBackoffMax; i++ { + wait *= 2 + } + s.until = now.Add(min(wait, f.lim.FullCheckBackoffMax)) +} + +// ReportVerifiedBundle records that a bundle from publisher p passed its full +// local check, which clears any full-check penalty of that publisher. +func (f *Fetcher) ReportVerifiedBundle(nw netid.NetworkID, p identity.NodeID) { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.fcStrikes, strikeKey(nw, p)) +} diff --git a/internal/peernet/bundlex/fetch_test.go b/internal/peernet/bundlex/fetch_test.go new file mode 100644 index 000000000..c6e95c606 --- /dev/null +++ b/internal/peernet/bundlex/fetch_test.go @@ -0,0 +1,397 @@ +package bundlex + +import ( + "bytes" + "context" + "errors" + "net/http" + "sync" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/database" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/contentid" + "github.com/kipsilabs/altmount/internal/peernet/identity" + "github.com/kipsilabs/altmount/internal/peernet/netid" + "github.com/kipsilabs/altmount/internal/peernet/recsync" + "github.com/kipsilabs/altmount/internal/peernet/transport" +) + +type fakeOffers struct { + rows []database.PeernetBundleOffer + atts []database.PeernetAttestation +} + +func (f *fakeOffers) BundleOffers(context.Context, []string, string, int) ([]database.PeernetBundleOffer, error) { + return f.rows, nil +} +func (f *fakeOffers) AttestationsFor(context.Context, string, string, string, time.Time) ([]database.PeernetAttestation, error) { + return f.atts, nil +} + +type fakeBook struct { + peers map[identity.NodeID]recsync.Peer + known map[string]bool +} + +func (f *fakeBook) VerifiedPeer(_ context.Context, _ netid.NetworkID, n identity.NodeID) (recsync.Peer, bool, error) { + p, ok := f.peers[n] + return p, ok, nil +} +func (f *fakeBook) Known(context.Context, netid.NetworkID, []string) (map[string]bool, error) { + return f.known, nil +} + +type fakeRecv struct{} + +func (fakeRecv) ReceivingNetworks() []recsync.NetworkInfo { + return []recsync.NetworkInfo{{ID: nwA, Level: 1}} +} + +// hostile starts a peer server with one custom handler on the bundle route. +func hostile(t *testing.T, h http.HandlerFunc) (string, identity.NodeID) { + t.Helper() + key, _ := identity.GenerateKey() + ts, err := transport.NewServer(transport.ServerConfig{Identity: key, ListenAddr: "127.0.0.1:0"}) + if err != nil { + t.Fatal(err) + } + if err := ts.Handle("GET "+PathBundle, 0, h); err != nil { + t.Fatal(err) + } + if err := ts.Start(context.Background()); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + c, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + _ = ts.Stop(c) + }) + return ts.Addr().String(), key.NodeID() +} + +type fetchEnv struct { + f *Fetcher + offers *fakeOffers + book *fakeBook + built *Built + rel string +} + +func newFetchEnv(t *testing.T, lim FetchLimits) *fetchEnv { + t.Helper() + lib, rel := nzbRelease() + built, err := newTestBuilder(t, lib, NoPatches{}).Build(context.Background(), rel, "T", posted) + if err != nil { + t.Fatal(err) + } + ck, _ := identity.GenerateKey() + client, err := transport.NewClient(transport.ClientConfig{Identity: ck, AllowLoopbackForTesting: true}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(client.Close) + e := &fetchEnv{offers: &fakeOffers{}, book: &fakeBook{peers: map[identity.NodeID]recsync.Peer{}, known: map[string]bool{}}, built: built, rel: rel} + e.f = NewFetcher(FetchConfig{Self: ck.NodeID(), Store: e.offers, Peers: e.book, Networks: fakeRecv{}, Dialer: client, Limits: lim}) + return e +} + +// publisher registers a node as a verified peer that announced the bundle. +func (e *fetchEnv) publisher(addr string, id identity.NodeID, age time.Duration) { + e.book.peers[id] = recsync.Peer{NodeID: id, Address: addr, Trusted: true} + e.offers.rows = append(e.offers.rows, database.PeernetBundleOffer{NetworkID: nwA.String(), PublisherNodeID: id.String(), + BundleID: e.built.ID.String(), ReleaseScheme: database.PeernetSchemeNZB, CreatedAt: time.Now().Add(-age), Source: database.PeernetRecordSourcePeer}) +} + +func honest(t *testing.T, e *fetchEnv) (string, identity.NodeID) { + return hostile(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", itoa(len(e.built.Wire))) + _, _ = w.Write(e.built.Wire) + }) +} + +func TestFetchVerifiesAndReportsSource(t *testing.T) { + e := newFetchEnv(t, FetchLimits{}) + addr, id := honest(t, e) + e.publisher(addr, id, time.Hour) + b, src, err := e.f.Fetch(context.Background(), e.rel) + if err != nil { + t.Fatal(err) + } + rid, _ := contentid.ParseID(e.rel) + if b.ReleaseID != rid || src.Publisher != id || src.Network != nwA || src.BundleID != e.built.ID || src.Scheme != bundle.SchemeNZB { + t.Fatalf("bundle %v source %+v", b.ReleaseID, src) + } + if e.f.Status().OK != 1 { + t.Fatal("success not counted") + } +} + +func TestFetchRejectsSubstitutedGarbageOversizedAndStrikes(t *testing.T) { + // A valid bundle of another release, served under the announced IDs. + lib2 := &fakeLib{scheme: database.PeernetSchemeNZB, paths: []string{"x"}, nzb: [][]string{ids("z", 4)}} + lib2.files = map[string]*metapb.FileMetadata{"x": plainFile("z.mkv", ids("z", 4))} + other, err := newTestBuilder(t, lib2, NoPatches{}).Build(context.Background(), releaseHexOfNZB(lib2.nzb), "T", posted) + if err != nil { + t.Fatal(err) + } + cases := map[string]struct { + h http.HandlerFunc + reason string + }{ + "substituted": {func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(other.Wire) }, failVerification}, + "garbage": {func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(bytes.Repeat([]byte{0x42}, 5000)) }, failVerification}, + "oversized": {func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", itoa(bundle.MaxCompressedSize+1)) + _, _ = w.Write(make([]byte, 1000)) + }, failTooLarge}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + e := newFetchEnv(t, FetchLimits{}) + bad, badID := hostile(t, c.h) + good, goodID := honest(t, e) + e.publisher(bad, badID, time.Minute) // newest: tried first + e.publisher(good, goodID, time.Hour) + b, src, err := e.f.Fetch(context.Background(), e.rel) + if err != nil || src.Publisher != goodID || b == nil { + t.Fatalf("the honest offer was not used: %v %+v", err, src) + } + st := e.f.Status() + if st.Failed[c.reason] != 1 || st.BackedOff != 1 { + t.Fatalf("status %+v", st) + } + // The hostile publisher is skipped from now on, the honest one still works. + e.f.glob = newBuckets(3600, 100, 1) + if _, src, err := e.f.Fetch(context.Background(), e.rel); err != nil || src.Publisher != goodID { + t.Fatalf("second fetch: %v %+v", err, src) + } + if e.f.Status().Failed[c.reason] != 1 { + t.Fatal("the struck publisher was asked again") + } + }) + } +} + +func TestFetchTruncatedAndSlowAreSoftFailures(t *testing.T) { + e := newFetchEnv(t, FetchLimits{Timeout: 400 * time.Millisecond}) + trunc, tid := hostile(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", itoa(len(e.built.Wire))) + _, _ = w.Write(e.built.Wire[:len(e.built.Wire)/2]) + }) + slow, sid := hostile(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", itoa(len(e.built.Wire))) + w.WriteHeader(200) + w.(http.Flusher).Flush() + time.Sleep(3 * time.Second) + }) + e.publisher(trunc, tid, time.Minute) + e.publisher(slow, sid, time.Hour) + _, _, err := e.f.Fetch(context.Background(), e.rel) + if !errors.Is(err, ErrAllFailed) { + t.Fatalf("err = %v", err) + } + if st := e.f.Status(); st.Failed[failTruncated] != 2 { + t.Fatalf("status %+v", st) + } +} + +func TestFetchOnlyFromVerifiedPublishers(t *testing.T) { + e := newFetchEnv(t, FetchLimits{}) + addr, id := honest(t, e) + e.publisher(addr, id, time.Hour) + delete(e.book.peers, id) // relayed record: no verified address for the publisher + if _, _, err := e.f.Fetch(context.Background(), e.rel); !errors.Is(err, ErrNoOffers) { + t.Fatalf("err = %v", err) + } + offers, _ := e.f.Offers(context.Background(), e.rel) + if len(offers) != 1 || offers[0].Fetchable { + t.Fatalf("offers %+v", offers) + } + // An own record is never fetched. + e.offers.rows[0].Source = database.PeernetRecordSourceLocal + e.book.peers[id] = recsync.Peer{NodeID: id, Address: addr, Trusted: true} + if _, _, err := e.f.Fetch(context.Background(), e.rel); !errors.Is(err, ErrNoOffers) { + t.Fatalf("own offer: %v", err) + } +} + +func TestOffersAreOrderedKnownThenAttestedThenNewest(t *testing.T) { + e := newFetchEnv(t, FetchLimits{}) + var ids3 [4]identity.NodeID + for i := range ids3 { + k, _ := identity.GenerateKey() + ids3[i] = k.NodeID() + e.publisher("127.0.0.1:1", ids3[i], time.Duration(i+1)*time.Hour) // 0 is newest + } + e.book.known[ids3[3].String()] = true + e.offers.atts = []database.PeernetAttestation{{PublisherNodeID: ids3[2].String(), Healthy: true}, {PublisherNodeID: ids3[1].String(), Healthy: false}} + got, err := e.f.Offers(context.Background(), e.rel) + if err != nil { + t.Fatal(err) + } + // ids3[2] attested only itself: a self-attestation never raises its own + // rank, while the others are attested by it. + want := []identity.NodeID{ids3[3], ids3[0], ids3[1], ids3[2]} + for i, o := range got { + if o.Publisher != want[i] { + t.Fatalf("position %d: got %v", i, o.Publisher) + } + } +} + +func TestFetchRateLimitsAndAttemptCap(t *testing.T) { + e := newFetchEnv(t, FetchLimits{GlobalPerHour: 1, GlobalBurst: 1}) + addr, id := honest(t, e) + e.publisher(addr, id, time.Hour) + if _, _, err := e.f.Fetch(context.Background(), e.rel); err != nil { + t.Fatal(err) + } + if _, _, err := e.f.Fetch(context.Background(), e.rel); !errors.Is(err, ErrRateLimited) { + t.Fatalf("second fetch: %v", err) + } + + // Attempt cap: five dead publishers, three attempts. + e = newFetchEnv(t, FetchLimits{MaxAttempts: 3}) + var mu sync.Mutex + hits := 0 + for range 5 { + a, i := hostile(t, func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + hits++ + mu.Unlock() + w.WriteHeader(500) + }) + e.publisher(a, i, time.Hour) + } + if _, _, err := e.f.Fetch(context.Background(), e.rel); !errors.Is(err, ErrAllFailed) { + t.Fatal(err) + } + if hits != 3 { + t.Fatalf("%d attempts", hits) + } +} + +func TestFetchConcurrencyCap(t *testing.T) { + e := newFetchEnv(t, FetchLimits{MaxConcurrent: 1}) + gate := make(chan struct{}) + addr, id := hostile(t, func(w http.ResponseWriter, _ *http.Request) { + <-gate + _, _ = w.Write(e.built.Wire) + }) + e.publisher(addr, id, time.Hour) + done := make(chan error, 1) + go func() { _, _, err := e.f.Fetch(context.Background(), e.rel); done <- err }() + time.Sleep(200 * time.Millisecond) + if _, _, err := e.f.Fetch(context.Background(), e.rel); !errors.Is(err, ErrFetchBusy) { + t.Fatalf("err = %v", err) + } + close(gate) + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestFullCheckPenaltyIsMildBoundedAndCleared(t *testing.T) { + e := newFetchEnv(t, FetchLimits{}) + addr, id := honest(t, e) + e.publisher(addr, id, time.Hour) + now := time.Now() + e.f.cfg.Now = func() time.Time { return now } + fetch := func() error { + e.f.glob = newBuckets(3600, 100, 1) + e.f.pub = newBuckets(3600, 100, 8) + _, _, err := e.f.Fetch(context.Background(), e.rel) + return err + } + if err := fetch(); err != nil { + t.Fatal(err) + } + // First report: 30 minutes. + e.f.ReportBadBundle(nwA, id) + if err := fetch(); !errors.Is(err, ErrNoOffers) { + t.Fatalf("a penalised publisher was asked: %v", err) + } + st := e.f.Status() + if st.FullCheckReports != 1 || st.FullCheckBackedOff != 1 { + t.Fatalf("status %+v", st) + } + if len(st.Failed) != 0 || st.BackedOff != 0 { + t.Fatalf("the penalty must not look like a hostile strike: %+v", st) + } + now = now.Add(31 * time.Minute) + if err := fetch(); err != nil { + t.Fatalf("not released after 30 minutes: %v", err) + } + // Second report doubles it. + e.f.ReportBadBundle(nwA, id) + now = now.Add(31 * time.Minute) + if err := fetch(); !errors.Is(err, ErrNoOffers) { + t.Fatal("the second penalty must last longer than 31 minutes") + } + now = now.Add(30 * time.Minute) + if err := fetch(); err != nil { + t.Fatalf("not released after 60 minutes: %v", err) + } + // Many reports stay under the cap. + for range 20 { + e.f.ReportBadBundle(nwA, id) + } + now = now.Add(6*time.Hour + time.Minute) + if err := fetch(); err != nil { + t.Fatalf("the penalty is not capped at 6 hours: %v", err) + } + // A later verified bundle clears it at once. + e.f.ReportBadBundle(nwA, id) + e.f.ReportVerifiedBundle(nwA, id) + if err := fetch(); err != nil { + t.Fatalf("a good full check must clear the penalty: %v", err) + } + // A long quiet period decays the doubling back to the base. + e.f.ReportBadBundle(nwA, id) + e.f.ReportBadBundle(nwA, id) + now = now.Add(25 * time.Hour) + e.f.ReportBadBundle(nwA, id) + now = now.Add(31 * time.Minute) + if err := fetch(); err != nil { + t.Fatalf("quiet period must reset the doubling: %v", err) + } +} + +func TestOffersPreferBundlesAnnouncedByMostDistinctNetworkRanges(t *testing.T) { + e := newFetchEnv(t, FetchLimits{}) + mk := func() identity.NodeID { k, _ := identity.GenerateKey(); return k.NodeID() } + announce := func(id identity.NodeID, addr, bundleID string, age time.Duration) { + e.book.known[id.String()] = true + e.book.peers[id] = recsync.Peer{NodeID: id, Address: addr, Trusted: true} + e.offers.rows = append(e.offers.rows, database.PeernetBundleOffer{NetworkID: nwA.String(), PublisherNodeID: id.String(), + BundleID: bundleID, ReleaseScheme: database.PeernetSchemeNZB, CreatedAt: time.Now().Add(-age), Source: database.PeernetRecordSourcePeer}) + } + x, y, z := e.built.ID.String(), "ab"+e.built.ID.String()[2:], "cd"+e.built.ID.String()[2:] + loner, a, b, d1, d2 := mk(), mk(), mk(), mk(), mk() + announce(loner, "10.9.9.1:8470", y, time.Minute) // newest, but announced by one publisher + announce(a, "10.1.1.5:8470", x, time.Hour) + announce(b, "10.2.2.5:8470", x, 2*time.Hour) + announce(d1, "10.3.3.5:8470", z, 3*time.Hour) // same /24: counts once + announce(d2, "10.3.3.77:8470", z, 4*time.Hour) + + got, err := e.f.Offers(context.Background(), e.rel) + if err != nil { + t.Fatal(err) + } + by := map[identity.NodeID]Offer{} + for _, o := range got { + by[o.Publisher] = o + } + if by[a].Corroborators != 2 || by[b].Corroborators != 2 { + t.Fatalf("two distinct ranges announce bundle x: got %d and %d", by[a].Corroborators, by[b].Corroborators) + } + if by[loner].Corroborators != 1 || by[d1].Corroborators != 1 || by[d2].Corroborators != 1 { + t.Fatalf("one range counts once: loner %d, same-range %d and %d", by[loner].Corroborators, by[d1].Corroborators, by[d2].Corroborators) + } + if got[0].Publisher != a || got[1].Publisher != b { + t.Fatalf("the corroborated bundle must rank first, got %v then %v", got[0].Publisher, got[1].Publisher) + } +} diff --git a/internal/peernet/bundlex/helpers_test.go b/internal/peernet/bundlex/helpers_test.go new file mode 100644 index 000000000..42e291ec2 --- /dev/null +++ b/internal/peernet/bundlex/helpers_test.go @@ -0,0 +1,119 @@ +package bundlex + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/database" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +const segSize = 716800 + +// marker strings that must never appear in a bundle. +var privateMarkers = []string{ + "/local/store/path", "RCLONE-PASSWORD-MARK", "RCLONE-SALT-MARK", "download-id-MARK", "IndexerMARK", + "CategoryMARK", "ArrNameMARK", "provider.mark.example", "health-history-MARK", "fingerprint-MARK", +} + +func ids(prefix string, n int) []string { + out := make([]string, n) + for i := range out { + out[i] = fmt.Sprintf("%s-%06d@news.test", prefix, i) + } + return out +} + +// plainFile is a healthy, unencrypted file whose segments are all of segs. +func plainFile(name string, segs []string) *metapb.FileMetadata { + m := &metapb.FileMetadata{ + FileSize: int64(len(segs)) * segSize, SourceNzbPath: "/local/store/path/x.nzbz", StoreRef: "/local/store/path/ref.nzbz", + Status: metapb.FileStatus_FILE_STATUS_HEALTHY, CreatedAt: 1, ModifiedAt: 2, ReleaseDate: 1700000000, + OriginalFilename: name, NzbdavId: "download-id-MARK", Password: "", Salt: "", + HoleProviderFingerprint: "fingerprint-MARK", + } + for _, id := range segs { + m.SegmentData = append(m.SegmentData, &metapb.SegmentData{Id: id, SegmentSize: segSize, StartOffset: 0, EndOffset: segSize - 1}) + } + return m +} + +// fakeLib implements Library and BuildSource over in-memory data. +type fakeLib struct { + mu sync.Mutex + scheme int + paths []string + files map[string]*metapb.FileMetadata + nzb [][]string + pwNZB map[string]string + reads int + readErr error + slowRead time.Duration +} + +func (f *fakeLib) ReleaseFiles(context.Context, string) (int, []string, error) { + return f.scheme, f.paths, nil +} + +func (f *fakeLib) ReadFile(p string) (*metapb.FileMetadata, error) { + f.mu.Lock() + f.reads++ + err, slow := f.readErr, f.slowRead + m := f.files[p] + f.mu.Unlock() + if slow > 0 { + time.Sleep(slow) + } + return m, err +} + +func (f *fakeLib) NZBSegmentIDs(string) ([][]string, error) { return f.nzb, nil } + +func (f *fakeLib) NZBPassword(p string, _ *metapb.FileMetadata) (string, bool) { + pw, ok := f.pwNZB[p] + return pw, ok +} + +type patches map[string]bool + +func (p patches) Has(id string) bool { return p[id] } + +// nzbRelease is a two-file NZB release (one video, one par2 not imported). +func nzbRelease() (*fakeLib, string) { + video := ids("vid", 20) + par := ids("par", 3) + lib := &fakeLib{scheme: database.PeernetSchemeNZB, paths: []string{"movies/R/movie.mkv"}, + files: map[string]*metapb.FileMetadata{"movies/R/movie.mkv": plainFile("movie.mkv", video)}, + nzb: [][]string{video, par}} + return lib, releaseHexOfNZB(lib.nzb) +} + +func releaseHexOfNZB(files [][]string) string { + store := &metapb.NzbStore{} + for _, f := range files { + fe := &metapb.NzbFileEntry{} + for _, id := range f { + fe.Segments = append(fe.Segments, &metapb.NzbSeg{Id: id}) + } + store.Files = append(store.Files, fe) + } + id, err := contentid.ReleaseID(store) + if err != nil { + panic(err) + } + return id.String() +} + +var posted = time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC) + +func newTestBuilder(t testing.TB, lib *fakeLib, p PatchIndex) *Builder { + t.Helper() + return NewBuilder(BuilderConfig{Source: lib, Library: lib, Patches: p}) +} + +func (f *fakeLib) setReadErr(err error) { f.mu.Lock(); f.readErr = err; f.mu.Unlock() } +func (f *fakeLib) readCount() int { f.mu.Lock(); defer f.mu.Unlock(); return f.reads } diff --git a/internal/peernet/bundlex/limits.go b/internal/peernet/bundlex/limits.go new file mode 100644 index 000000000..e126e4130 --- /dev/null +++ b/internal/peernet/bundlex/limits.go @@ -0,0 +1,140 @@ +package bundlex + +import "time" + +// Wire constants. +const ( + // PathBundle is the serve route: GET with query n (network) and b (Bundle ID). + PathBundle = "/peer/v1/bundle" + queryNet = "n" + queryBundle = "b" +) + +// ServeLimits bound what this node serves. Zero fields take the defaults. +// Numbers are design calls: a bundle is up to 8 MB, so a handful an hour per +// peer is plenty for on-demand imports. +type ServeLimits struct { + // PerPeerPerHour and PerPeerBurst: bundle requests per authenticated peer. + PerPeerPerHour, PerPeerBurst int + // PerIPPerHour and PerIPBurst: requests per public remote address. Private, + // loopback and link-local addresses are shared by honest nodes of one LAN + // and are not limited per address. + PerIPPerHour, PerIPBurst int + // BytesPerMinute and BytesBurst: node-wide budget of bundle bytes sent. + BytesPerMinute, BytesBurst int64 + // MaxConcurrent, MaxConcurrentPerPeer and MaxConcurrentPerIP cap bundles + // being served at once. + MaxConcurrent, MaxConcurrentPerPeer, MaxConcurrentPerIP int + // ChunkDeadline bounds each write; TotalDeadline the whole response. + ChunkDeadline, TotalDeadline time.Duration + // MaxTracked bounds the memory of the per-peer and per-address accounting. + MaxTracked int +} + +// DefaultServeLimits returns the defaults. +func DefaultServeLimits() ServeLimits { + return ServeLimits{ + PerPeerPerHour: 30, PerPeerBurst: 5, + PerIPPerHour: 60, PerIPBurst: 10, + BytesPerMinute: 128 << 20, BytesBurst: 32 << 20, + MaxConcurrent: 4, MaxConcurrentPerPeer: 1, MaxConcurrentPerIP: 2, + ChunkDeadline: 15 * time.Second, TotalDeadline: 60 * time.Second, + MaxTracked: 4096, + } +} + +func (l ServeLimits) withDefaults() ServeLimits { + d := DefaultServeLimits() + i := func(v *int, def int) { + if *v <= 0 { + *v = def + } + } + i(&l.PerPeerPerHour, d.PerPeerPerHour) + i(&l.PerPeerBurst, d.PerPeerBurst) + i(&l.PerIPPerHour, d.PerIPPerHour) + i(&l.PerIPBurst, d.PerIPBurst) + i(&l.MaxConcurrent, d.MaxConcurrent) + i(&l.MaxConcurrentPerPeer, d.MaxConcurrentPerPeer) + i(&l.MaxConcurrentPerIP, d.MaxConcurrentPerIP) + i(&l.MaxTracked, d.MaxTracked) + if l.BytesPerMinute <= 0 { + l.BytesPerMinute = d.BytesPerMinute + } + if l.BytesBurst <= 0 { + l.BytesBurst = d.BytesBurst + } + if l.ChunkDeadline <= 0 { + l.ChunkDeadline = d.ChunkDeadline + } + if l.TotalDeadline <= 0 { + l.TotalDeadline = d.TotalDeadline + } + return l +} + +// FetchLimits bound what this node downloads. Fetching happens only on demand +// from a local caller, never speculatively. +type FetchLimits struct { + // MaxAttempts is how many offers one Fetch tries. + MaxAttempts int + // GlobalPerHour/GlobalBurst and PerPublisherPerHour/PerPublisherBurst are + // download attempts. + GlobalPerHour, GlobalBurst int + PerPublisherPerHour, PerPublisherBurst int + // MaxConcurrent caps downloads at once. + MaxConcurrent int + // Timeout bounds one download including the body. + Timeout time.Duration + // BackoffBase and BackoffMax: hostile publishers are skipped for a time that + // doubles per strike. SoftBackoff is the skip after an ordinary failure. + BackoffBase, BackoffMax, SoftBackoff time.Duration + // FullCheckBackoffBase, FullCheckBackoffMax and FullCheckDecay shape the mild + // penalty of a publisher whose verified bundle failed the local full check. + FullCheckBackoffBase, FullCheckBackoffMax, FullCheckDecay time.Duration + // MaxTracked bounds the accounting tables. + MaxTracked int + // MaxOffers bounds the offers read from the store. + MaxOffers int +} + +// DefaultFetchLimits returns the defaults. +func DefaultFetchLimits() FetchLimits { + return FetchLimits{ + MaxAttempts: 3, GlobalPerHour: 60, GlobalBurst: 10, PerPublisherPerHour: 12, PerPublisherBurst: 3, + MaxConcurrent: 2, Timeout: 60 * time.Second, + BackoffBase: 10 * time.Minute, BackoffMax: 2 * time.Hour, SoftBackoff: 5 * time.Minute, + FullCheckBackoffBase: 30 * time.Minute, FullCheckBackoffMax: 6 * time.Hour, FullCheckDecay: 24 * time.Hour, + MaxTracked: 2048, MaxOffers: 32, + } +} + +func (l FetchLimits) withDefaults() FetchLimits { + d := DefaultFetchLimits() + i := func(v *int, def int) { + if *v <= 0 { + *v = def + } + } + dur := func(v *time.Duration, def time.Duration) { + if *v <= 0 { + *v = def + } + } + i(&l.MaxAttempts, d.MaxAttempts) + i(&l.GlobalPerHour, d.GlobalPerHour) + i(&l.GlobalBurst, d.GlobalBurst) + i(&l.PerPublisherPerHour, d.PerPublisherPerHour) + i(&l.PerPublisherBurst, d.PerPublisherBurst) + i(&l.MaxConcurrent, d.MaxConcurrent) + i(&l.MaxTracked, d.MaxTracked) + i(&l.MaxOffers, d.MaxOffers) + dur(&l.Timeout, d.Timeout) + dur(&l.BackoffBase, d.BackoffBase) + dur(&l.BackoffMax, d.BackoffMax) + dur(&l.SoftBackoff, d.SoftBackoff) + dur(&l.FullCheckBackoffBase, d.FullCheckBackoffBase) + dur(&l.FullCheckBackoffMax, d.FullCheckBackoffMax) + dur(&l.FullCheckDecay, d.FullCheckDecay) + return l +} diff --git a/internal/peernet/bundlex/serve.go b/internal/peernet/bundlex/serve.go new file mode 100644 index 000000000..659d53bc0 --- /dev/null +++ b/internal/peernet/bundlex/serve.go @@ -0,0 +1,336 @@ +package bundlex + +import ( + "context" + "encoding/hex" + "errors" + "io" + "net/http" + "sync" + "time" + + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/discovery" + "github.com/kipsilabs/altmount/internal/peernet/identity" + "github.com/kipsilabs/altmount/internal/peernet/netid" + "github.com/kipsilabs/altmount/internal/peernet/publish" + "github.com/kipsilabs/altmount/internal/peernet/recsync" + "github.com/kipsilabs/altmount/internal/peernet/transport" +) + +// Announcement is what this node remembers of a bundle it announced. +type Announcement struct { + ReleaseID string + // OriginCategory and OriginIndexer are the private origin tags kept with + // the publish state, used to re-check exclusions at serve time. + OriginCategory, OriginIndexer string +} + +// Announcements is this node's record of what it announced. +type Announcements interface { + // Announced finds the release this node announced in nw with Bundle ID id. + Announced(ctx context.Context, nw netid.NetworkID, id bundle.BundleID) (Announcement, bool, error) + // Withdraw deletes this node's records and publish state of the release. + Withdraw(ctx context.Context, nw netid.NetworkID, releaseHex string) error + // MarkStale makes the release's record be announced again. + MarkStale(ctx context.Context, nw netid.NetworkID, releaseHex string) error +} + +// Provider returns the packed bundle of a release (building it if needed). +type Provider interface { + Bundle(ctx context.Context, releaseHex string) (*Built, error) +} + +// NetworkLevels reports the exposure level of a registered network. +type NetworkLevels interface { + NetworkLevel(id netid.NetworkID) (int, bool) +} + +// ServerConfig wires the serve route. +type ServerConfig struct { + Networks NetworkLevels + Auth discovery.Authorizer + Announcements Announcements + Provider Provider + Exclusions func() publish.Exclusions + // BackedOff reports a peer or address record exchange has struck. + BackedOff func(nw netid.NetworkID, peer identity.NodeID, ip string) bool + Limits ServeLimits + Now func() time.Time + Log func(ctx context.Context, key string, err error) +} + +// Server serves the bundles this node announced. +type Server struct { + cfg ServerConfig + lim ServeLimits + peers *buckets + ips *buckets + bytes *buckets + perPer *counter + perIP *counter + slots chan struct{} + + mu sync.Mutex + served int64 + sentByte int64 + refused map[string]int64 +} + +// NewServer returns a Server. +func NewServer(cfg ServerConfig) *Server { + if cfg.Now == nil { + cfg.Now = time.Now + } + if cfg.Auth == nil { + cfg.Auth = discovery.AllowAll{} + } + if cfg.Log == nil { + cfg.Log = func(context.Context, string, error) {} + } + l := cfg.Limits.withDefaults() + s := &Server{cfg: cfg, lim: l, + peers: newBuckets(l.PerPeerPerHour, l.PerPeerBurst, l.MaxTracked), + ips: newBuckets(l.PerIPPerHour, l.PerIPBurst, l.MaxTracked), + perPer: newCounter(), perIP: newCounter(), + slots: make(chan struct{}, l.MaxConcurrent), refused: map[string]int64{}} + // The byte budget refills per minute; express it per hour for the bucket. + s.bytes = &buckets{rate: float64(l.BytesPerMinute) / 60, burst: float64(l.BytesBurst), max: 1, m: map[string]*bucket{}} + return s +} + +// RegisterRoutes adds the bundle route (no request body allowed). +func (s *Server) RegisterRoutes(r recsync.Registrar) error { + return r.Handle("GET "+PathBundle, 0, http.HandlerFunc(s.handle)) +} + +// ServeStatus counts what was served since start. +type ServeStatus struct { + Served, Bytes int64 + Refused map[string]int64 +} + +// Status returns the counters. +func (s *Server) Status() ServeStatus { + s.mu.Lock() + defer s.mu.Unlock() + ref := make(map[string]int64, len(s.refused)) + for k, v := range s.refused { + ref[k] = v + } + return ServeStatus{Served: s.served, Bytes: s.sentByte, Refused: ref} +} + +func (s *Server) refuse(reason string) { + s.mu.Lock() + s.refused[reason]++ + s.mu.Unlock() +} + +func text(w http.ResponseWriter, dec discovery.ResponseDecorator, code int, msg string) { + h := w.Header() + if dec != nil { + dec(h) + } + h.Set("Content-Type", "text/plain; charset=utf-8") + h.Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(code) + _, _ = io.WriteString(w, msg+"\n") +} + +// deny is the one answer for every refusal that must not tell the caller why: +// level below 3, unknown network, denied caller, unknown or unavailable bundle. +func (s *Server) deny(w http.ResponseWriter, reason string) { + s.refuse(reason) + text(w, nil, http.StatusForbidden, "forbidden") +} + +func (s *Server) handle(w http.ResponseWriter, r *http.Request) { + peer, ok := transport.PeerFromContext(r.Context()) + if !ok { + s.deny(w, "unauthenticated") + return + } + q := r.URL.Query() + nv, bv := q[queryNet], q[queryBundle] + if len(nv) != 1 || len(bv) != 1 || len(q) != 2 { + text(w, nil, http.StatusBadRequest, "bad request") + return + } + nw, err := netid.ParseNetworkID(nv[0]) + var bid bundle.BundleID + raw, herr := hex.DecodeString(bv[0]) + if err != nil || herr != nil || len(raw) != len(bid) || !lowerHex(bv[0]) { + text(w, nil, http.StatusBadRequest, "bad request") + return + } + copy(bid[:], raw) + + level, known := s.cfg.Networks.NetworkLevel(nw) + if !known || !recsync.PolicyFor(level).ServeBundles { + s.deny(w, "level") + return + } + dec, err := s.cfg.Auth.AuthorizeInbound(r.Context(), nw, peer, r.Header.Clone()) + if err != nil { + s.deny(w, "denied") + return + } + ip := "" + if a, ok := transport.RemoteIPFromContext(r.Context()); ok { + ip = a.String() + } + if s.cfg.BackedOff != nil && s.cfg.BackedOff(nw, peer, ip) { + s.deny(w, "backed_off") + return + } + // From here the caller is a member: tell it about load, not about content. + now := s.cfg.Now() + if !s.peers.take(peer.String(), 1, now) || (ip != "" && !sharedIP(ip) && !s.ips.take(ip, 1, now)) { + s.refuse("rate_limited") + text(w, dec, http.StatusTooManyRequests, "rate limited") + return + } + release, ok := s.admit(peer.String(), ip) + if !ok { + s.refuse("busy") + text(w, dec, http.StatusTooManyRequests, "busy") + return + } + defer release() + + ann, found, err := s.cfg.Announcements.Announced(r.Context(), nw, bid) + if err != nil { + s.cfg.Log(r.Context(), "bundle announcement lookup", err) + s.refuse("error") + text(w, dec, http.StatusServiceUnavailable, "unavailable") + return + } + if !found { + s.deny(w, "unknown_bundle") + return + } + if s.cfg.Exclusions != nil && s.cfg.Exclusions().CoversOrigin(ann.OriginCategory, ann.OriginIndexer) { + if err := s.cfg.Announcements.Withdraw(r.Context(), nw, ann.ReleaseID); err != nil { + s.cfg.Log(r.Context(), "withdraw excluded release", err) + } + s.deny(w, "excluded") + return + } + built, err := s.cfg.Provider.Bundle(r.Context(), ann.ReleaseID) + switch { + case errors.Is(err, ErrBusy): + s.refuse("busy") + text(w, dec, http.StatusServiceUnavailable, "busy") + return + case err != nil || built == nil || built.ID != bid: + // Not buildable any more, or not the bytes that were announced (the + // library changed): never serve other bytes; announce again. + if err == nil { + _ = s.cfg.Announcements.MarkStale(r.Context(), nw, ann.ReleaseID) + } + s.deny(w, "unavailable") + return + } + if !s.bytes.take("all", float64(len(built.Wire)), now) { + s.refuse("byte_budget") + text(w, dec, http.StatusTooManyRequests, "busy") + return + } + s.send(w, dec, built.Wire) +} + +func lowerHex(s string) bool { + for i := 0; i < len(s); i++ { + c := s[i] + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +// admit takes a global slot and the per-peer and per-address slots. +func (s *Server) admit(peer, ip string) (func(), bool) { + select { + case s.slots <- struct{}{}: + default: + return nil, false + } + if !s.perPer.enter(peer, s.lim.MaxConcurrentPerPeer) { + <-s.slots + return nil, false + } + useIP := ip != "" && !sharedIP(ip) + if useIP && !s.perIP.enter(ip, s.lim.MaxConcurrentPerIP) { + s.perPer.leave(peer) + <-s.slots + return nil, false + } + return func() { + if useIP { + s.perIP.leave(ip) + } + s.perPer.leave(peer) + <-s.slots + }, true +} + +const chunkSize = 32 << 10 + +// send streams body with an exact Content-Length, a deadline per chunk and a +// total deadline. Range requests are not supported: the header is ignored and +// the whole body is sent. The body is stored compressed and never re-encoded. +func (s *Server) send(w http.ResponseWriter, dec discovery.ResponseDecorator, body []byte) { + h := w.Header() + if dec != nil { + dec(h) + } + h.Set("Content-Type", "application/octet-stream") + h.Set("Content-Length", itoa(len(body))) + h.Set("Cache-Control", "no-store") + h.Set("X-Content-Type-Options", "nosniff") + h.Set("Accept-Ranges", "none") + h.Del("Content-Encoding") + w.WriteHeader(http.StatusOK) + rc := http.NewResponseController(w) + start := s.cfg.Now() + sent := 0 + for sent < len(body) { + dl := start.Add(s.lim.TotalDeadline) + if c := s.cfg.Now().Add(s.lim.ChunkDeadline); c.Before(dl) { + dl = c + } + _ = rc.SetWriteDeadline(dl) + n := min(chunkSize, len(body)-sent) + if _, err := w.Write(body[sent : sent+n]); err != nil { + break + } + sent += n + if s.cfg.Now().Sub(start) > s.lim.TotalDeadline { + break + } + } + s.mu.Lock() + s.sentByte += int64(sent) + if sent == len(body) { + s.served++ + } else { + s.refused["aborted"]++ + } + s.mu.Unlock() +} + +func itoa(n int) string { + var b [20]byte + i := len(b) + if n == 0 { + return "0" + } + for n > 0 { + i-- + b[i] = byte('0' + n%10) + n /= 10 + } + return string(b[i:]) +} diff --git a/internal/peernet/bundlex/serve_test.go b/internal/peernet/bundlex/serve_test.go new file mode 100644 index 000000000..62b820e1f --- /dev/null +++ b/internal/peernet/bundlex/serve_test.go @@ -0,0 +1,334 @@ +package bundlex + +import ( + "bytes" + "context" + "errors" + "net/http" + "net/url" + "sync" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/discovery" + "github.com/kipsilabs/altmount/internal/peernet/identity" + "github.com/kipsilabs/altmount/internal/peernet/netid" + "github.com/kipsilabs/altmount/internal/peernet/publish" + "github.com/kipsilabs/altmount/internal/peernet/transport" +) + +var ( + nwA = netid.NetworkID{0xa} + nwB = netid.NetworkID{0xb} + nwC = netid.NetworkID{0xc, 0xc} +) + +type fakeNets struct { + mu sync.Mutex + m map[netid.NetworkID]int +} + +func (f *fakeNets) NetworkLevel(id netid.NetworkID) (int, bool) { + f.mu.Lock() + defer f.mu.Unlock() + l, ok := f.m[id] + return l, ok +} +func (f *fakeNets) set(id netid.NetworkID, l int) { f.mu.Lock(); f.m[id] = l; f.mu.Unlock() } + +type fakeAnn struct { + mu sync.Mutex + byNet map[netid.NetworkID]map[bundle.BundleID]Announcement + withdrew []string + stale []string +} + +func (f *fakeAnn) Announced(_ context.Context, nw netid.NetworkID, id bundle.BundleID) (Announcement, bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + a, ok := f.byNet[nw][id] + return a, ok, nil +} +func (f *fakeAnn) Withdraw(_ context.Context, _ netid.NetworkID, rel string) error { + f.mu.Lock() + f.withdrew = append(f.withdrew, rel) + f.mu.Unlock() + return nil +} +func (f *fakeAnn) MarkStale(_ context.Context, _ netid.NetworkID, rel string) error { + f.mu.Lock() + f.stale = append(f.stale, rel) + f.mu.Unlock() + return nil +} + +type fakeProvider struct { + built *Built + err error + gate chan struct{} + calls int + mu sync.Mutex +} + +func (f *fakeProvider) Bundle(context.Context, string) (*Built, error) { + f.mu.Lock() + f.calls++ + f.mu.Unlock() + if f.gate != nil { + <-f.gate + } + return f.built, f.err +} + +type denyAuth struct{ discovery.AllowAll } + +func (denyAuth) AuthorizeInbound(context.Context, netid.NetworkID, identity.NodeID, http.Header) (discovery.ResponseDecorator, error) { + return nil, errors.New("not a member") +} + +type serveEnv struct { + srv *Server + nets *fakeNets + ann *fakeAnn + prov *fakeProvider + excl *publish.Exclusions + client *transport.Client + addr string + id identity.NodeID + built *Built +} + +func newServeEnv(t *testing.T, mut func(*ServerConfig)) *serveEnv { + t.Helper() + lib, rel := nzbRelease() + built, err := newTestBuilder(t, lib, NoPatches{}).Build(context.Background(), rel, "T", posted) + if err != nil { + t.Fatal(err) + } + e := &serveEnv{nets: &fakeNets{m: map[netid.NetworkID]int{nwA: 3}}, prov: &fakeProvider{built: built}, + ann: &fakeAnn{byNet: map[netid.NetworkID]map[bundle.BundleID]Announcement{nwA: {built.ID: {ReleaseID: rel, OriginCategory: "movies", OriginIndexer: "idx"}}}}, + excl: &publish.Exclusions{}, built: built} + cfg := ServerConfig{Networks: e.nets, Announcements: e.ann, Provider: e.prov, Exclusions: func() publish.Exclusions { return *e.excl }} + if mut != nil { + mut(&cfg) + } + e.srv = NewServer(cfg) + key, err := identity.GenerateKey() + if err != nil { + t.Fatal(err) + } + ts, err := transport.NewServer(transport.ServerConfig{Identity: key, ListenAddr: "127.0.0.1:0"}) + if err != nil { + t.Fatal(err) + } + if err := e.srv.RegisterRoutes(ts); err != nil { + t.Fatal(err) + } + if err := ts.Start(context.Background()); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ctx, c := context.WithTimeout(context.Background(), 3*time.Second) + defer c() + _ = ts.Stop(ctx) + }) + e.addr, e.id = ts.Addr().String(), key.NodeID() + ck, _ := identity.GenerateKey() + e.client, err = transport.NewClient(transport.ClientConfig{Identity: ck, AllowLoopbackForTesting: true}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(e.client.Close) + return e +} + +func (e *serveEnv) get(t *testing.T, nw netid.NetworkID, b string, hdr http.Header) (int, http.Header, []byte) { + t.Helper() + return e.getFrom(t, e.client, nw, b, hdr) +} + +func (e *serveEnv) getFrom(t *testing.T, c *transport.Client, nw netid.NetworkID, b string, hdr http.Header) (int, http.Header, []byte) { + t.Helper() + resp, err := c.Do(context.Background(), e.addr, e.id, transport.Request{ + Method: http.MethodGet, Path: PathBundle, Query: url.Values{"n": {nw.String()}, "b": {b}}, Header: hdr, + }, transport.CallOptions{MaxResponseBytes: bundle.MaxCompressedSize, TrustedAddress: true, Timeout: 10 * time.Second}) + if err != nil { + t.Fatal(err) + } + body, err := resp.ReadAll() + if err != nil { + t.Fatal(err) + } + return resp.StatusCode, resp.Header, body +} + +func TestServeSendsExactBytesWithoutRangeOrCompression(t *testing.T) { + e := newServeEnv(t, nil) + hdr := http.Header{"Range": {"bytes=10-20"}, "Accept-Encoding": {"gzip, br"}} + code, h, body := e.get(t, nwA, e.built.ID.String(), hdr) + if code != 200 || !bytes.Equal(body, e.built.Wire) { + t.Fatalf("status %d, %d bytes", code, len(body)) + } + if h.Get("Content-Length") != itoa(len(e.built.Wire)) || h.Get("Content-Encoding") != "" || h.Get("Accept-Ranges") != "none" { + t.Fatalf("headers %v", h) + } + st := e.srv.Status() + if st.Served != 1 || st.Bytes != int64(len(e.built.Wire)) { + t.Fatalf("status %+v", st) + } +} + +func TestServeRefusalsAreIdentical(t *testing.T) { + e := newServeEnv(t, nil) + e.nets.set(nwC, 2) + // The bundle is announced in nwC too, but nwC is only at level 2. + e.ann.byNet[nwC] = map[bundle.BundleID]Announcement{e.built.ID: {ReleaseID: e.built.ReleaseID}} + e.nets.set(nwB, 3) + unknown := bytes.Repeat([]byte{7}, 32) + cases := map[string]func() (int, []byte){ + "unknown network": func() (int, []byte) { c, _, b := e.get(t, netid.NetworkID{0xc}, e.built.ID.String(), nil); return c, b }, + "level 2": func() (int, []byte) { c, _, b := e.get(t, nwC, e.built.ID.String(), nil); return c, b }, + "unknown bundle": func() (int, []byte) { c, _, b := e.get(t, nwA, bundle.BundleID(unknown).String(), nil); return c, b }, + "announced only in another network": func() (int, []byte) { + c, _, b := e.get(t, nwB, e.built.ID.String(), nil) + return c, b + }, + } + var wantCode int + var wantBody []byte + for name, f := range cases { + code, body := f() + if code != http.StatusForbidden { + t.Errorf("%s: status %d", name, code) + } + if wantBody == nil { + wantCode, wantBody = code, body + } + if code != wantCode || !bytes.Equal(body, wantBody) { + t.Errorf("%s: answer differs: %d %q", name, code, body) + } + } + if e.prov.calls != 0 { + t.Fatal("a refused request caused a build") + } +} + +func TestServeDeniedCallerAndBackedOffPeerAreRefused(t *testing.T) { + e := newServeEnv(t, func(c *ServerConfig) { c.Auth = denyAuth{} }) + if code, _, body := e.get(t, nwA, e.built.ID.String(), nil); code != 403 || e.prov.calls != 0 { + t.Fatalf("denied caller: %d %q", code, body) + } + e2 := newServeEnv(t, func(c *ServerConfig) { + c.BackedOff = func(netid.NetworkID, identity.NodeID, string) bool { return true } + }) + if code, _, _ := e2.get(t, nwA, e2.built.ID.String(), nil); code != 403 { + t.Fatalf("backed off peer: %d", code) + } +} + +func TestServeRechecksExclusionsAndWithdraws(t *testing.T) { + e := newServeEnv(t, nil) + e.excl.Indexers = []string{"IDX"} + if code, _, _ := e.get(t, nwA, e.built.ID.String(), nil); code != 403 { + t.Fatalf("excluded release served: %d", code) + } + if len(e.ann.withdrew) != 1 || e.prov.calls != 0 { + t.Fatalf("withdrew %v, builds %d", e.ann.withdrew, e.prov.calls) + } +} + +func TestServeNeverServesDifferentBytesThanAnnounced(t *testing.T) { + e := newServeEnv(t, nil) + other := *e.built + other.ID[0] ^= 0xff + e.prov.built = &other + if code, _, _ := e.get(t, nwA, e.built.ID.String(), nil); code != 403 { + t.Fatalf("status %d", code) + } + if len(e.ann.stale) != 1 { + t.Fatalf("stale %v", e.ann.stale) + } + e.prov.built, e.prov.err = nil, ErrBusy + if code, _, _ := e.get(t, nwA, e.built.ID.String(), nil); code != 503 { + t.Fatalf("busy status %d", code) + } +} + +func TestServePerPeerRateLimit(t *testing.T) { + e := newServeEnv(t, func(c *ServerConfig) { c.Limits = ServeLimits{PerPeerPerHour: 1, PerPeerBurst: 3} }) + for i := range 3 { + if code, _, _ := e.get(t, nwA, e.built.ID.String(), nil); code != 200 { + t.Fatalf("request %d: %d", i, code) + } + } + if code, _, _ := e.get(t, nwA, e.built.ID.String(), nil); code != http.StatusTooManyRequests { + t.Fatalf("over the burst: %d", code) + } + // Another peer has its own allowance (loopback: no per-address limit). + ck, _ := identity.GenerateKey() + c2, err := transport.NewClient(transport.ClientConfig{Identity: ck, AllowLoopbackForTesting: true}) + if err != nil { + t.Fatal(err) + } + defer c2.Close() + if code, _, _ := e.getFrom(t, c2, nwA, e.built.ID.String(), nil); code != 200 { + t.Fatalf("second peer: %d", code) + } +} + +func TestServeNodeWideByteBudget(t *testing.T) { + var size int64 + e := newServeEnv(t, nil) + size = int64(len(e.built.Wire)) + e = newServeEnv(t, func(c *ServerConfig) { c.Limits = ServeLimits{BytesBurst: size*2 + 1, BytesPerMinute: 1} }) + for i := range 2 { + if code, _, _ := e.get(t, nwA, e.built.ID.String(), nil); code != 200 { + t.Fatalf("request %d: %d", i, code) + } + } + if code, _, _ := e.get(t, nwA, e.built.ID.String(), nil); code != http.StatusTooManyRequests { + t.Fatalf("over the byte budget: %d", code) + } +} + +func TestServeConcurrencyPerPeer(t *testing.T) { + e := newServeEnv(t, nil) + e.prov.gate = make(chan struct{}) + done := make(chan int, 1) + go func() { c, _, _ := e.get(t, nwA, e.built.ID.String(), nil); done <- c }() + deadline := time.Now().Add(3 * time.Second) + for { + e.prov.mu.Lock() + n := e.prov.calls + e.prov.mu.Unlock() + if n == 1 || time.Now().After(deadline) { + break + } + time.Sleep(10 * time.Millisecond) + } + if code, _, _ := e.get(t, nwA, e.built.ID.String(), nil); code != http.StatusTooManyRequests { + t.Fatalf("second concurrent request from one peer: %d", code) + } + close(e.prov.gate) + if c := <-done; c != 200 { + t.Fatalf("first request: %d", c) + } +} + +func TestServeMalformedRequests(t *testing.T) { + e := newServeEnv(t, nil) + for _, b := range []string{"zz", "ABCDEF", e.built.ID.String()[:62]} { + if code, _, _ := e.get(t, nwA, b, nil); code != http.StatusBadRequest { + t.Errorf("%q: %d", b, code) + } + } +} + +func TestSharedIPClassification(t *testing.T) { + for ip, want := range map[string]bool{"10.1.2.3": true, "127.0.0.1": true, "192.168.0.9": true, "fe80::1": true, "8.8.8.8": false, "2001:4860::1": false} { + if sharedIP(ip) != want { + t.Errorf("%s: shared = %v", ip, !want) + } + } +} diff --git a/internal/peernet/bundlex/source.go b/internal/peernet/bundlex/source.go new file mode 100644 index 000000000..5d06feea7 --- /dev/null +++ b/internal/peernet/bundlex/source.go @@ -0,0 +1,142 @@ +// Package bundlex builds, serves and fetches peer-network bundles (level 3). +// +// The builder reads local data only through the narrow BuildSource and +// Library interfaces defined here; nothing else in the peernet packages +// receives full file metadata. The serve route answers a single request +// shape (one network, one Bundle ID this node announced there) and the fetch +// client downloads from the publisher of a signed release record only, then +// verifies the bytes with bundle.Verify against the IDs of the signed record. +package bundlex + +import ( + "context" + "strings" + + "github.com/kipsilabs/altmount/internal/metadata" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +// BuildSource is the builder's only view of local import data. It is the one +// place the peernet code reads full per-file metadata (segments, layout and +// encryption fields); keep it out of every other package's reach. +type BuildSource interface { + // ReadFile returns the fully resolved metadata of a virtual file, or + // (nil, nil) when it is absent. + ReadFile(virtualPath string) (*metapb.FileMetadata, error) + // NZBSegmentIDs returns the message-IDs of every NZB file of the release + // the file belongs to, in NZB order. It returns (nil, nil) when the file + // has no release store (or only one the metadata migration rebuilt), so + // the release has no NZB segment list. No store path is exposed. + NZBSegmentIDs(virtualPath string) ([][]string, error) + // NZBPassword reports whether the key material of the file derives from a + // password present inside the NZB content itself (a meta password), and + // returns that password. A password spelled in the NZB file name, given by + // the user, an ARR or the API, or one whose origin is not recorded, is not + // from the NZB. + NZBPassword(virtualPath string, m *metapb.FileMetadata) (password string, fromNZB bool) +} + +// PatchIndex says whether a local PAR2 patch exists for an article. +type PatchIndex interface { + Has(messageID string) bool +} + +// NoPatches is a PatchIndex for a node without a patch store. +type NoPatches struct{} + +// Has implements PatchIndex. +func (NoPatches) Has(string) bool { return false } + +// Library lists the indexed files of a release. +type Library interface { + // ReleaseFiles returns the stored scheme (database.PeernetScheme*) and the + // virtual paths of the release's files, one per distinct File ID. + ReleaseFiles(ctx context.Context, releaseHex string) (scheme int, paths []string, err error) +} + +// PasswordOrigins says whether a file's archive password came with the NZB's +// own content (the importer recorded it at import; see +// database.PeernetPasswordOriginRepository). Anything it does not know is "no". +type PasswordOrigins interface { + FromNZB(ctx context.Context, fileID string) (bool, error) +} + +type metadataSource struct { + ms *metadata.MetadataService + origins PasswordOrigins +} + +// NewMetadataSource adapts the metadata service to BuildSource. +// +// The metadata records the AES key and IV of an archive file but not the +// password they were derived from, nor where it came from. The importer +// records, for new imports only, that a file's password came with the NZB's own +// content (origins); every other file, including all imports from before that +// record existed, is "not from the NZB", and a release with key material is +// then not shareable. +func NewMetadataSource(ms *metadata.MetadataService, origins PasswordOrigins) BuildSource { + return metadataSource{ms: ms, origins: origins} +} + +func (s metadataSource) ReadFile(p string) (*metapb.FileMetadata, error) { + return s.ms.ReadFileMetadata(p) +} + +func (s metadataSource) NZBSegmentIDs(p string) ([][]string, error) { + m, err := s.ms.ReadFileMetadata(p) + if err != nil || m == nil { + return nil, err + } + if m.StoreRef == "" || migratedStore(m.StoreRef) { + return nil, nil + } + store, err := s.ms.Store().ReadStore(m.StoreRef) + if err != nil || store == nil { + return nil, err + } + out := make([][]string, len(store.Files)) + for i, f := range store.Files { + if f == nil { + continue + } + ids := make([]string, len(f.Segments)) + for j, sg := range f.Segments { + if sg != nil { + ids[j] = sg.Id + } + } + out[i] = ids + } + return out, nil +} + +// NZBPassword answers from the origin the importer recorded for the file's +// File ID (so a rename or move does not lose it). The password text is never +// stored anywhere, so the answer is always ("", fromNZB): the bundle then +// carries the key and IV, which is all a receiver needs to play the release. +// A file with no recorded origin, a lookup error or no origin store is not +// from the NZB. +func (s metadataSource) NZBPassword(_ string, m *metapb.FileMetadata) (string, bool) { + if s.origins == nil || m == nil { + return "", false + } + id, err := contentid.FileID(m) + if err != nil { + return "", false + } + from, err := s.origins.FromNZB(context.Background(), id.String()) + if err != nil || !from { + return "", false + } + return "", true +} + +func migratedStore(ref string) bool { + for _, part := range strings.Split(strings.ReplaceAll(ref, "\\", "/"), "/") { + if part == "_migrated" { + return true + } + } + return false +} diff --git a/internal/peernet/bundlex/source_test.go b/internal/peernet/bundlex/source_test.go new file mode 100644 index 000000000..8c1d63b6a --- /dev/null +++ b/internal/peernet/bundlex/source_test.go @@ -0,0 +1,113 @@ +package bundlex + +import ( + "bytes" + "context" + "errors" + "testing" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/metadata" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +type fakeOrigins struct { + from map[string]bool + err error +} + +func (f fakeOrigins) FromNZB(_ context.Context, id string) (bool, error) { return f.from[id], f.err } + +// keyedFile is an AES-keyed file whose segments cover its padded size. +func keyedFile(segs []string) *metapb.FileMetadata { + m := plainFile("movie.mkv", segs) + m.FileSize -= 5 + m.StoreRef, m.SourceNzbPath = "", "" + m.Encryption, m.AesKey, m.AesIv = metapb.Encryption_AES, bytes.Repeat([]byte{0x41}, 32), bytes.Repeat([]byte{0x42}, 16) + return m +} + +type pathLib struct{ paths []string } + +func (l pathLib) ReleaseFiles(context.Context, string) (int, []string, error) { + return database.PeernetSchemeFiles, l.paths, nil +} + +func fileIDOf(t *testing.T, m *metapb.FileMetadata) string { + t.Helper() + id, err := contentid.FileID(m) + if err != nil { + t.Fatal(err) + } + return id.String() +} + +func TestMetadataSourcePasswordOriginComesFromTheRecordedFlag(t *testing.T) { + ms := metadata.NewMetadataService(t.TempDir()) + m := keyedFile(ids("k", 3)) + if err := ms.WriteFileMetadata("movies/R/movie.mkv", m); err != nil { + t.Fatal(err) + } + read, err := ms.ReadFileMetadata("movies/R/movie.mkv") + if err != nil { + t.Fatal(err) + } + flagged := fakeOrigins{from: map[string]bool{fileIDOf(t, read): true}} + + if pw, ok := NewMetadataSource(ms, flagged).NZBPassword("movies/R/movie.mkv", read); !ok || pw != "" { + t.Fatalf("flagged file: (%q, %v); the password itself is never known", pw, ok) + } + if _, ok := NewMetadataSource(ms, fakeOrigins{}).NZBPassword("movies/R/movie.mkv", read); ok { + t.Fatal("an unflagged file (every pre-existing import) must not count as from the NZB") + } + if _, ok := NewMetadataSource(ms, fakeOrigins{from: flagged.from, err: errors.New("db down")}).NZBPassword("p", read); ok { + t.Fatal("a failed lookup must fail closed") + } + if _, ok := NewMetadataSource(ms, nil).NZBPassword("p", read); ok { + t.Fatal("no origin store: never from the NZB") + } + // A move or rename changes the path, not the File ID: the flag follows. + if err := ms.WriteFileMetadata("tv/Moved/other-name.mkv", m); err != nil { + t.Fatal(err) + } + moved, _ := ms.ReadFileMetadata("tv/Moved/other-name.mkv") + if _, ok := NewMetadataSource(ms, flagged).NZBPassword("tv/Moved/other-name.mkv", moved); !ok { + t.Fatal("the flag did not survive a move") + } +} + +func TestKeyedReleaseBuildsOnlyWithTheRecordedFlag(t *testing.T) { + ms := metadata.NewMetadataService(t.TempDir()) + if err := ms.WriteFileMetadata("movies/R/movie.mkv", keyedFile(ids("k", 3))); err != nil { + t.Fatal(err) + } + read, _ := ms.ReadFileMetadata("movies/R/movie.mkv") + fid, _ := contentid.ParseID(fileIDOf(t, read)) + rel, err := contentid.ReleaseIDFromFiles([]contentid.ID{fid}) + if err != nil { + t.Fatal(err) + } + build := func(o PasswordOrigins) (*Built, error) { + b := NewBuilder(BuilderConfig{Source: NewMetadataSource(ms, o), Library: pathLib{paths: []string{"movies/R/movie.mkv"}}, Patches: NoPatches{}}) + return b.Build(context.Background(), rel.String(), "T", posted) + } + if _, err := build(fakeOrigins{}); notShareableReason(t, err) != "password_not_from_nzb" { + t.Fatalf("without the flag: %v", err) + } + built, err := build(fakeOrigins{from: map[string]bool{fid.String(): true}}) + if err != nil { + t.Fatalf("with the flag: %v", err) + } + got, err := bundle.Verify(built.Wire, built.ID, rel) + if err != nil { + t.Fatal(err) + } + if len(got.Files) != 1 || !bytes.Equal(got.Files[0].AESKey, bytes.Repeat([]byte{0x41}, 32)) || !bytes.Equal(got.Files[0].AESIV, bytes.Repeat([]byte{0x42}, 16)) { + t.Fatalf("the key and IV must travel: %+v", got.Files) + } + if got.NZBPassword != "" { + t.Fatal("the password text is not known to the node and must not appear") + } +} diff --git a/internal/peernet/contentid/bundle_identity.go b/internal/peernet/contentid/bundle_identity.go new file mode 100644 index 000000000..f7dd8d8e2 --- /dev/null +++ b/internal/peernet/contentid/bundle_identity.go @@ -0,0 +1,51 @@ +package contentid + +import ( + "crypto/sha256" + "encoding/hex" + + "github.com/kipsilabs/altmount/internal/database" +) + +// A file imported from a peer bundle has no source NZB path, so the usual +// group key (a hash of that path) is empty and the usual derivation cannot see +// it. The bundle's Release ID, however, was verified cryptographically by the +// receiver against the segment lists it was given, so it is recorded directly +// in the index, under a group key derived locally from that very ID. A row +// whose group key is the key of its own release ID is a "verified bundle row". +// No schema change is needed: when the derive pass later changes the ID (the +// user deleted a file of a "files" release), the row's key no longer matches +// its ID and it is an ordinary derived row, exactly as for a local import. + +// bundleGroupKey is the local-only group key of a bundle's files. It is never +// transmitted and is not part of any ID. +func bundleGroupKey(releaseHex string) string { + sum := sha256.Sum256([]byte("altmount-peernet/bundle-group/v1\x00" + releaseHex)) + return hex.EncodeToString(sum[:]) +} + +// VerifiedEntry returns the index row of a file whose release identity came +// from a verified peer bundle: its File ID, the bundle's Release ID and scheme +// (database.PeernetSchemeNZB or PeernetSchemeFiles) and the local group key. +func VerifiedEntry(filePath string, fileID, releaseID ID, scheme int) database.PeernetContentEntry { + return database.PeernetContentEntry{ + FilePath: filePath, FileID: fileID.String(), ReleaseID: releaseID.String(), + ReleaseScheme: scheme, GroupKey: bundleGroupKey(releaseID.String()), + } +} + +// IsVerifiedBundleRow reports whether the row still carries the Release ID a +// verified bundle gave it. +func IsVerifiedBundleRow(e database.PeernetContentEntry) bool { + return e.ReleaseID != "" && e.GroupKey != "" && e.GroupKey == bundleGroupKey(e.ReleaseID) +} + +// keepsBundleGrouping reports that a freshly computed entry e (which can only +// see the metadata) must not replace the stored row old: the file is the same +// and the row's group was assigned locally from a verified bundle, which the +// metadata (no source NZB path, no store) cannot reproduce. Once the derive +// pass has changed such a row's ID the row stays in its group and is derived +// like any other; only a changed file identity re-indexes it. +func keepsBundleGrouping(old, e database.PeernetContentEntry) bool { + return old.FileID == e.FileID && old.GroupKey != "" && e.GroupKey == "" && e.ReleaseScheme != database.PeernetSchemeNZB +} diff --git a/internal/peernet/contentid/bundle_identity_test.go b/internal/peernet/contentid/bundle_identity_test.go new file mode 100644 index 000000000..c9026817f --- /dev/null +++ b/internal/peernet/contentid/bundle_identity_test.go @@ -0,0 +1,131 @@ +package contentid + +import ( + "context" + "testing" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// bundleFiles writes two files the way a bundle import does (no source NZB, no +// store) and records them in the index as verified, the way Accept does. +func bundleFiles(ctx context.Context, t *testing.T, f *fixture, scheme int, rel ID) []string { + t.Helper() + paths := []string{"peer/Rel/a.mkv", "peer/Rel/b.mkv"} + f.old(ctx, paths[0], "", "pa1", "pa2") + f.old(ctx, paths[1], "", "pb1") + for _, p := range paths { + require.NoError(t, f.repo.Upsert(ctx, VerifiedEntry(p, f.fileID(p), rel, scheme))) + } + return paths +} + +func filesRel(f *fixture, paths ...string) ID { + rid, err := ReleaseIDFromFiles([]ID{f.fileID(paths[0]), f.fileID(paths[1])}) + require.NoError(f.t, err) + return rid +} + +func TestVerifiedBundleRows_SurviveIndexReconcileAndDerive(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + paths := []string{"peer/Rel/a.mkv", "peer/Rel/b.mkv"} + f.old(ctx, paths[0], "", "pa1", "pa2") + f.old(ctx, paths[1], "", "pb1") + rel := filesRel(f, paths...) + bundleFiles(ctx, t, f, database.PeernetSchemeFiles, rel) + + row := f.row(ctx, paths[0]) + require.Equal(t, rel.String(), row.ReleaseID) + require.Equal(t, database.PeernetSchemeFiles, row.ReleaseScheme) + assert.NotEmpty(t, row.GroupKey, "a local group key derived from the release id") + assert.True(t, IsVerifiedBundleRow(row)) + + // The observer re-indexes on every metadata write: the verified row stays. + for _, p := range paths { + require.NoError(t, f.idx.IndexPath(ctx, p)) + } + assert.Equal(t, row, f.row(ctx, paths[0])) + + // Restart: backfill + reconcile + derive keep it. + _, err := f.idx.Backfill(ctx) + require.NoError(t, err) + res, err := f.idx.Reconcile(ctx) + require.NoError(t, err) + assert.Zero(t, res.Reindexed+res.Pruned, "reconcile must not drop a verified id") + dr, err := f.idx.DeriveAll(ctx) + require.NoError(t, err) + assert.Zero(t, dr.Assigned, "the derived id equals the verified one: nothing to write") + assert.Equal(t, row, f.row(ctx, paths[0])) +} + +func TestVerifiedBundleRows_FilesSchemeChangesWhenAFileIsDeleted(t *testing.T) { + ctx := context.Background() + o := &orphans{} + f := newFixture(t, WithOrphanHandler(o.handle)) + paths := []string{"peer/Rel/a.mkv", "peer/Rel/b.mkv"} + f.old(ctx, paths[0], "", "pa1", "pa2") + f.old(ctx, paths[1], "", "pb1") + rel := filesRel(f, paths...) + bundleFiles(ctx, t, f, database.PeernetSchemeFiles, rel) + + require.NoError(t, f.ms.DeleteFileMetadata(ctx, paths[1])) + NewObserver(f.idx).FileDeleted(ctx, paths[1]) + _, err := f.idx.DeriveDirty(ctx) + require.NoError(t, err) + + left := f.row(ctx, paths[0]) + want, err := ReleaseIDFromFiles([]ID{f.fileID(paths[0])}) + require.NoError(t, err) + assert.Equal(t, want.String(), left.ReleaseID, "exactly as for a local import: the group id follows its files") + assert.False(t, IsVerifiedBundleRow(left), "now derived, no longer verified") + assert.Equal(t, []string{rel.String()}, o.list(), "the old id is withdrawn") + + // A restart after that changes nothing. + _, err = f.idx.Reconcile(ctx) + require.NoError(t, err) + dr, err := f.idx.DeriveAll(ctx) + require.NoError(t, err) + assert.Zero(t, dr.Assigned) + assert.Equal(t, want.String(), f.row(ctx, paths[0]).ReleaseID) +} + +func TestVerifiedBundleRows_NZBSchemeIDOutlivesADeletedFile(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + paths := []string{"peer/Rel/a.mkv", "peer/Rel/b.mkv"} + f.old(ctx, paths[0], "", "pa1", "pa2") + f.old(ctx, paths[1], "", "pb1") + rel := f.fileID(paths[0]) // any 32 bytes stand in for the NZB-scheme release id + bundleFiles(ctx, t, f, database.PeernetSchemeNZB, rel) + + _, err := f.idx.Backfill(ctx) + require.NoError(t, err) + res, err := f.idx.Reconcile(ctx) + require.NoError(t, err) + assert.Zero(t, res.Reindexed+res.Pruned) + assert.Equal(t, rel.String(), f.row(ctx, paths[0]).ReleaseID) + assert.Equal(t, database.PeernetSchemeNZB, f.row(ctx, paths[0]).ReleaseScheme) + + require.NoError(t, f.ms.DeleteFileMetadata(ctx, paths[1])) + NewObserver(f.idx).FileDeleted(ctx, paths[1]) + _, err = f.idx.DeriveDirty(ctx) + require.NoError(t, err) + assert.Equal(t, rel.String(), f.row(ctx, paths[0]).ReleaseID, "an NZB-scheme id names the NZB, not the file set") +} + +func TestVerifiedBundleRows_AChangedFileIsReindexedNormally(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + paths := []string{"peer/Rel/a.mkv", "peer/Rel/b.mkv"} + f.old(ctx, paths[0], "", "pa1", "pa2") + f.old(ctx, paths[1], "", "pb1") + bundleFiles(ctx, t, f, database.PeernetSchemeFiles, filesRel(f, paths...)) + f.old(ctx, paths[0], "", "other1") // the file's content identity changed + res, err := f.idx.Reconcile(ctx) + require.NoError(t, err) + assert.Equal(t, 1, res.Reindexed) + assert.False(t, IsVerifiedBundleRow(f.row(ctx, paths[0]))) +} diff --git a/internal/peernet/contentid/derive.go b/internal/peernet/contentid/derive.go new file mode 100644 index 000000000..3ba67970d --- /dev/null +++ b/internal/peernet/contentid/derive.go @@ -0,0 +1,464 @@ +package contentid + +import ( + "context" + "fmt" + "log/slog" + "sort" + "sync" + "sync/atomic" + + "github.com/kipsilabs/altmount/internal/database" +) + +// Defaults of the derive pass. The pass is a keyset scan over the group index: +// memory is bounded by deriveRowBatch rows (a single group may exceed it, up to +// MaxDerivedReleaseFiles), and every write is a short transaction of at most one +// batch. +const ( + defaultDeriveGroups = 500 + defaultDeriveRows = 5000 + defaultMaxDirty = 50000 + maxPendingOrphans = 50000 + maxDirRefs = 20000 + progressEveryGroups = 20000 +) + +// DeriveResult summarises one derive pass. +type DeriveResult struct { + Groups int // groups examined + Assigned int // rows whose derived ID was set or changed + Cleared int // rows that lost a derived ID (group over the cap) + OversizeFiles int // files in groups over the cap (no ID) +} + +// deriveState is the incremental bookkeeping of the derive pass: which groups +// changed since the last pass, which derived IDs may have been superseded, and +// the knobs. +type deriveState struct { + run sync.Mutex // one pass at a time + + mu sync.Mutex + dirty map[string]struct{} + full bool + orphans map[string]struct{} + + maxGroup int + groupBatch int + rowBatch int + maxDirty int + onDirty func() + onOrphans func(ctx context.Context, releaseIDs []string) + + oversize atomic.Int64 +} + +func newDeriveState() *deriveState { + return &deriveState{ + dirty: map[string]struct{}{}, orphans: map[string]struct{}{}, + maxGroup: MaxDerivedReleaseFiles, groupBatch: defaultDeriveGroups, rowBatch: defaultDeriveRows, maxDirty: defaultMaxDirty, + } +} + +// WithMaxGroupFiles lowers the file-count cap of a derived release (it can +// never exceed MaxDerivedReleaseFiles). +func WithMaxGroupFiles(n int) Option { + return func(i *Indexer) { + if n > 0 && n <= MaxDerivedReleaseFiles { + i.derive.maxGroup = n + } + } +} + +// WithOrphanHandler registers the callback that receives derived release IDs +// that no indexed file carries any more (their group changed or vanished), so +// the records published under them can be withdrawn. It runs on the goroutine +// of the pass and must return quickly. +func WithOrphanHandler(fn func(ctx context.Context, releaseIDs []string)) Option { + return func(i *Indexer) { i.derive.onOrphans = fn } +} + +// WithDirtyNotify registers a non-blocking callback run when a group is marked +// for re-derivation, so an owner can schedule DeriveDirty (debounced). +func WithDirtyNotify(fn func()) Option { return func(i *Indexer) { i.derive.onDirty = fn } } + +// WithDeriveBatch sets how many groups one derive page lists and how many rows +// one members read may hold. Values below 1 are ignored. +func WithDeriveBatch(groups, rows int) Option { + return func(i *Indexer) { + if groups > 0 { + i.derive.groupBatch = groups + } + if rows > 0 { + i.derive.rowBatch = rows + } + } +} + +// WithMaxDirty bounds the number of remembered dirty groups; beyond it the +// next DeriveDirty does a full pass instead. +func WithMaxDirty(n int) Option { + return func(i *Indexer) { + if n > 0 { + i.derive.maxDirty = n + } + } +} + +// OversizeFiles is the number of files that the last full pass left without a +// release ID because their group is over the cap. +func (i *Indexer) OversizeFiles() int { return int(i.derive.oversize.Load()) } + +// HasDirty reports whether a DeriveDirty pass has work. +func (i *Indexer) HasDirty() bool { + d := i.derive + d.mu.Lock() + defer d.mu.Unlock() + return d.full || len(d.dirty) > 0 || len(d.orphans) > 0 +} + +func (d *deriveState) markDirty(key string) { + if key == "" { + return + } + d.mu.Lock() + switch { + case d.full: + case len(d.dirty) >= d.maxDirty: + d.full, d.dirty = true, map[string]struct{}{} + default: + d.dirty[key] = struct{}{} + } + notify := d.onDirty + d.mu.Unlock() + if notify != nil { + notify() + } +} + +func (d *deriveState) addOrphan(id string) { + if id == "" { + return + } + d.mu.Lock() + if len(d.orphans) < maxPendingOrphans { + d.orphans[id] = struct{}{} + } + d.mu.Unlock() +} + +// noteChanged records that old (the zero row when there was none) was replaced +// by e. +func (i *Indexer) noteChanged(old, e database.PeernetContentEntry) { + d := i.derive + if old.FilePath != "" { + if old.ReleaseScheme == database.PeernetSchemeFiles { + d.addOrphan(old.ReleaseID) + } + d.markDirty(old.GroupKey) + } + if e.ReleaseScheme != database.PeernetSchemeNZB { + d.markDirty(e.GroupKey) + } +} + +// noteRemoved records that a row with this identity left the index. +func (i *Indexer) noteRemoved(ref database.PeernetContentRef) { + d := i.derive + if ref.ReleaseScheme == database.PeernetSchemeFiles { + d.addOrphan(ref.ReleaseID) + } + d.markDirty(ref.GroupKey) +} + +// markAllDirty asks for a full pass. +func (i *Indexer) markAllDirty() { + d := i.derive + d.mu.Lock() + d.full, d.dirty = true, map[string]struct{}{} + notify := d.onDirty + d.mu.Unlock() + if notify != nil { + notify() + } +} + +func (d *deriveState) take() (keys []string, full bool, orphans []string) { + d.mu.Lock() + defer d.mu.Unlock() + for k := range d.dirty { + keys = append(keys, k) + } + for id := range d.orphans { + orphans = append(orphans, id) + } + full = d.full + d.dirty, d.orphans, d.full = map[string]struct{}{}, map[string]struct{}{}, false + sort.Strings(keys) + return keys, full, orphans +} + +func (d *deriveState) putBack(keys []string, full bool, orphans []string) { + d.mu.Lock() + defer d.mu.Unlock() + d.full = d.full || full + for _, k := range keys { + if len(d.dirty) < d.maxDirty { + d.dirty[k] = struct{}{} + } else { + d.full = true + } + } + for _, id := range orphans { + if len(d.orphans) < maxPendingOrphans { + d.orphans[id] = struct{}{} + } + } +} + +// DeriveDirty re-derives only the groups that changed since the last pass (the +// observer and reconcile mark them) and reports superseded IDs. With nothing +// dirty it does no database work. If too many groups changed, or a directory +// removal was too large to track, it runs a full pass instead. +func (i *Indexer) DeriveDirty(ctx context.Context) (DeriveResult, error) { + d := i.derive + d.run.Lock() + defer d.run.Unlock() + keys, full, orphans := d.take() + if len(keys) == 0 && !full && len(orphans) == 0 { + return DeriveResult{}, nil + } + cands := map[string]struct{}{} + for _, id := range orphans { + cands[id] = struct{}{} + } + var res DeriveResult + var err error + if full { + res, err = i.deriveAll(ctx, cands) + if err != nil { + d.putBack(keys, true, nil) // the pass did not finish: all of it is still owed + } + } else { + done := 0 + for done < len(keys) && err == nil { + if err = ctx.Err(); err != nil { + break + } + end := min(done+d.groupBatch, len(keys)) + var counts map[string]int + if counts, err = i.repo.GroupCounts(ctx, keys[done:end]); err != nil { + err = fmt.Errorf("count dirty groups: %w", err) + break + } + err = i.applyGroups(ctx, keys[done:end], counts, &res, cands) + if err == nil { + done = end + } + } + if err != nil { + d.putBack(keys[done:], false, nil) + } + } + if ferr := i.flushOrphans(ctx, cands); ferr != nil && err == nil { + err = ferr + } + return res, err +} + +// DeriveAll walks every import group of the index once and gives each group +// its derived Release ID (see ReleaseIDFromFiles). It reads only the index +// table: no metadata and no store is read. +// +// Cost: one keyset page of group counts per deriveGroupBatch groups and one +// members read per deriveRowBatch rows, so O(files) rows read in total and +// O(1) pages of memory; writes touch only rows whose ID changes, each batch in +// one short transaction. It can be cancelled between batches and run again: a +// second pass over an unchanged index writes nothing. +func (i *Indexer) DeriveAll(ctx context.Context) (DeriveResult, error) { + d := i.derive + d.run.Lock() + defer d.run.Unlock() + // A full pass covers everything that was dirty before it started. + keys, full, orphans := d.take() + cands := map[string]struct{}{} + for _, id := range orphans { + cands[id] = struct{}{} + } + res, err := i.deriveAll(ctx, cands) + if err != nil { + d.putBack(keys, full, nil) // unfinished: what it was to cover is still owed + } + if ferr := i.flushOrphans(ctx, cands); ferr != nil && err == nil { + err = ferr + } + return res, err +} + +func (i *Indexer) deriveAll(ctx context.Context, cands map[string]struct{}) (DeriveResult, error) { + d := i.derive + var res DeriveResult + after := "" + nextLog := progressEveryGroups + for { + if err := ctx.Err(); err != nil { + return res, err + } + page, err := i.repo.GroupSummaries(ctx, after, d.groupBatch) + if err != nil { + return res, fmt.Errorf("list groups: %w", err) + } + if len(page) == 0 { + break + } + after = page[len(page)-1].Key + keys := make([]string, len(page)) + counts := make(map[string]int, len(page)) + for j, g := range page { + keys[j], counts[g.Key] = g.Key, g.Files + } + if err := i.applyGroups(ctx, keys, counts, &res, cands); err != nil { + return res, err + } + if res.Groups >= nextLog { + slog.InfoContext(ctx, "Peernet library release id derivation progress", "groups", res.Groups, "assigned", res.Assigned) + nextLog = res.Groups + progressEveryGroups + } + } + d.oversize.Store(int64(res.OversizeFiles)) + if res.Assigned > 0 || res.Cleared > 0 { + slog.InfoContext(ctx, "Peernet library release ids derived", + "groups", res.Groups, "assigned", res.Assigned, "cleared", res.Cleared, "oversize_files", res.OversizeFiles) + } + return res, nil +} + +// applyGroups derives the groups in keys (counts says how many files each has; +// an absent or zero count is an empty group). Superseded derived IDs are added +// to cands. +func (i *Indexer) applyGroups(ctx context.Context, keys []string, counts map[string]int, res *DeriveResult, cands map[string]struct{}) error { + d := i.derive + var batch []string + rows := 0 + flush := func() error { + if len(batch) == 0 { + return nil + } + err := i.deriveBatch(ctx, batch, res, cands) + batch, rows = batch[:0], 0 + return err + } + for _, k := range keys { + n := counts[k] + switch { + case n == 0: + continue + case n > d.maxGroup: + res.Groups++ + res.OversizeFiles += n + old, err := i.repo.ClearDerivedGroup(ctx, k, 0) + for _, id := range old { + cands[id] = struct{}{} + } + if err != nil { + return fmt.Errorf("clear oversize group: %w", err) + } + continue + } + if rows+n > d.rowBatch && len(batch) > 0 { + if err := flush(); err != nil { + return err + } + } + batch = append(batch, k) + rows += n + } + return flush() +} + +// deriveBatch reads the members of the given groups and writes the IDs. +func (i *Indexer) deriveBatch(ctx context.Context, keys []string, res *DeriveResult, cands map[string]struct{}) error { + if err := ctx.Err(); err != nil { + return err + } + members, err := i.repo.GroupMembers(ctx, keys) + if err != nil { + return fmt.Errorf("read group members: %w", err) + } + var updates []database.PeernetDerivedUpdate + for start := 0; start < len(members); { + end := start + for end < len(members) && members[end].GroupKey == members[start].GroupKey { + end++ + } + group := members[start:end] + start = end + res.Groups++ + ids := make([]ID, 0, len(group)) + ok := true + for _, m := range group { + id, err := ParseID(m.FileID) + if err != nil { + ok = false + break + } + ids = append(ids, id) + } + if !ok { + continue + } + rid, err := ReleaseIDFromFiles(ids) + if err != nil { + continue // empty or over the cap: no ID; the caller routes those groups elsewhere + } + want := rid.String() + for _, m := range group { + if m.ReleaseScheme == database.PeernetSchemeFiles && m.ReleaseID != want { + cands[m.ReleaseID] = struct{}{} + } + if m.ReleaseID != want || m.ReleaseScheme != database.PeernetSchemeFiles { + updates = append(updates, database.PeernetDerivedUpdate{ + FilePath: m.FilePath, FileID: m.FileID, GroupKey: m.GroupKey, ReleaseID: want}) + } + } + } + if len(updates) == 0 { + return nil + } + n, err := i.repo.ApplyDerived(ctx, updates) + if err != nil { + return fmt.Errorf("write derived release ids: %w", err) + } + res.Assigned += n + return nil +} + +// flushOrphans hands the superseded IDs that no indexed file carries any more +// to the orphan handler. It runs even when the pass was cancelled. +func (i *Indexer) flushOrphans(ctx context.Context, cands map[string]struct{}) error { + d := i.derive + if len(cands) == 0 { + return nil + } + ids := make([]string, 0, len(cands)) + for id := range cands { + ids = append(ids, id) + } + sort.Strings(ids) + ctx = context.WithoutCancel(ctx) + live, err := i.repo.StillIndexed(ctx, ids) + if err != nil { + d.putBack(nil, false, ids) + return fmt.Errorf("check superseded release ids: %w", err) + } + var dead []string + for _, id := range ids { + if !live[id] { + dead = append(dead, id) + } + } + if len(dead) > 0 && d.onOrphans != nil { + d.onOrphans(ctx, dead) + } + return nil +} diff --git a/internal/peernet/contentid/derive_bench_test.go b/internal/peernet/contentid/derive_bench_test.go new file mode 100644 index 000000000..c7f5d8a32 --- /dev/null +++ b/internal/peernet/contentid/derive_bench_test.go @@ -0,0 +1,80 @@ +package contentid + +import ( + "context" + "crypto/sha256" + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/stretchr/testify/require" +) + +// synthetic fills a repository with `files` old-format rows in groups of +// `perGroup` files (every row has a file ID and a group key, no release ID). +func synthetic(tb testing.TB, files, perGroup int) *database.PeernetContentRepository { + tb.Helper() + db, err := database.NewDB(database.Config{DatabasePath: filepath.Join(tb.TempDir(), "db")}) + require.NoError(tb, err) + tb.Cleanup(func() { _ = db.Close() }) + repo := database.NewPeernetContentRepository(db.Connection(), database.DialectSQLite) + ctx := context.Background() + batch := make([]database.PeernetContentEntry, 0, 1000) + for n := range files { + fid := sha256.Sum256(fmt.Appendf(nil, "file-%d", n)) + gid := sha256.Sum256(fmt.Appendf(nil, "group-%d", n/perGroup)) + batch = append(batch, database.PeernetContentEntry{ + FilePath: fmt.Sprintf("complete/Release.%d/file.%d.mkv", n/perGroup, n), + FileID: fmt.Sprintf("%x", fid), GroupKey: fmt.Sprintf("%x", gid), + }) + if len(batch) == cap(batch) { + require.NoError(tb, repo.UpsertBatch(ctx, batch)) + batch = batch[:0] + } + } + require.NoError(tb, repo.UpsertBatch(ctx, batch)) + return repo +} + +// TestDeriveAll_200kRows measures a full derive pass over 200,000 rows +// (40,000 groups of 5) and a second, no-op pass, on a fresh SQLite database. +// The bound is generous: it exists to catch an accidental O(n^2), not to be a +// benchmark. The measured times are logged. +func TestDeriveAll_200kRows(t *testing.T) { + if testing.Short() { + t.Skip("200k-row measurement") + } + repo := synthetic(t, 200_000, 5) + idx := NewIndexer(nil, repo) + ctx := context.Background() + + start := time.Now() + res, err := idx.DeriveAll(ctx) + first := time.Since(start) + require.NoError(t, err) + require.Equal(t, 40_000, res.Groups) + require.Equal(t, 200_000, res.Assigned) + + start = time.Now() + res, err = idx.DeriveAll(ctx) + second := time.Since(start) + require.NoError(t, err) + require.Equal(t, 0, res.Assigned) + + t.Logf("derive 200k rows / 40k groups: first pass %v (200k rows written), unchanged pass %v", first, second) + require.Less(t, first, 2*time.Minute) +} + +func BenchmarkDeriveAll200k(b *testing.B) { + for b.Loop() { + b.StopTimer() + repo := synthetic(b, 200_000, 5) + idx := NewIndexer(nil, repo) + b.StartTimer() + if _, err := idx.DeriveAll(context.Background()); err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/peernet/contentid/derive_test.go b/internal/peernet/contentid/derive_test.go new file mode 100644 index 000000000..3062208de --- /dev/null +++ b/internal/peernet/contentid/derive_test.go @@ -0,0 +1,488 @@ +package contentid + +import ( + "context" + "fmt" + "sort" + "sync" + "testing" + + "github.com/kipsilabs/altmount/internal/database" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// old writes one file in the older metadata format: segments inline, no store +// reference, the source NZB path set. +func (f *fixture) old(ctx context.Context, p, src string, ids ...string) { + f.t.Helper() + m := &metapb.FileMetadata{FileSize: 10, SegmentData: sd(ids...), SourceNzbPath: src, Status: metapb.FileStatus_FILE_STATUS_HEALTHY} + require.NoError(f.t, f.ms.WriteFileMetadata(p, m)) +} + +func (f *fixture) row(ctx context.Context, p string) database.PeernetContentEntry { + f.t.Helper() + e, ok, err := f.repo.GetByPath(ctx, p) + require.NoError(f.t, err) + require.True(f.t, ok, p) + return e +} + +func (f *fixture) fileID(p string) ID { + f.t.Helper() + m, err := f.ms.ReadFileMetadata(p) + require.NoError(f.t, err) + return mustFile(f.t, m) +} + +func (f *fixture) wantFilesID(paths ...string) string { + f.t.Helper() + ids := make([]ID, len(paths)) + for i, p := range paths { + ids[i] = f.fileID(p) + } + rid, err := ReleaseIDFromFiles(ids) + require.NoError(f.t, err) + return rid.String() +} + +type orphans struct { + mu sync.Mutex + ids map[string]bool +} + +func (o *orphans) handle(_ context.Context, ids []string) { + o.mu.Lock() + defer o.mu.Unlock() + if o.ids == nil { + o.ids = map[string]bool{} + } + for _, id := range ids { + o.ids[id] = true + } +} + +func (o *orphans) list() []string { + o.mu.Lock() + defer o.mu.Unlock() + var out []string + for id := range o.ids { + out = append(out, id) + } + sort.Strings(out) + return out +} + +const ( + srcA = "/data/.nzbs/1/a.nzb.gz" + srcB = "/data/.nzbs/2/b.nzb.gz" + srcC = "/data/.nzbs/3/c.nzb.gz" +) + +func indexAll(ctx context.Context, t *testing.T, f *fixture) DeriveResult { + t.Helper() + _, err := f.idx.Backfill(ctx) + require.NoError(t, err) + res, err := f.idx.DeriveAll(ctx) + require.NoError(t, err) + return res +} + +// Files of one import are grouped by their source NZB, never by folder: a +// folder can hold several imports, and one import can span folders. +func TestDerive_GroupsByImportNotFolder(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + // Import A spans two folders (release folder and a nested sample folder). + f.old(ctx, "cat/RelA/a.mkv", srcA, "a1", "a2") + f.old(ctx, "cat/RelA/Sample/s.mkv", srcA, "s1") + // Imports B and C share one folder (a season folder with one NZB per episode). + f.old(ctx, "cat/Season/e1.mkv", srcB, "b1", "b2") + f.old(ctx, "cat/Season/e2.mkv", srcC, "c1") + // A single file directly in the category folder next to unrelated files. + f.old(ctx, "cat/lonely.mkv", "/data/.nzbs/9/lonely.nzb.gz", "l1") + + res := indexAll(ctx, t, f) + assert.Equal(t, 4, res.Groups) + + relA := f.wantFilesID("cat/RelA/a.mkv", "cat/RelA/Sample/s.mkv") + assert.Equal(t, relA, f.row(ctx, "cat/RelA/a.mkv").ReleaseID) + assert.Equal(t, relA, f.row(ctx, "cat/RelA/Sample/s.mkv").ReleaseID) + assert.Equal(t, f.wantFilesID("cat/Season/e1.mkv"), f.row(ctx, "cat/Season/e1.mkv").ReleaseID) + assert.Equal(t, f.wantFilesID("cat/Season/e2.mkv"), f.row(ctx, "cat/Season/e2.mkv").ReleaseID) + assert.NotEqual(t, f.row(ctx, "cat/Season/e1.mkv").ReleaseID, f.row(ctx, "cat/Season/e2.mkv").ReleaseID) + lonely := f.row(ctx, "cat/lonely.mkv") + assert.Equal(t, f.wantFilesID("cat/lonely.mkv"), lonely.ReleaseID, "a single file is a release of one file") + for _, p := range []string{"cat/RelA/a.mkv", "cat/Season/e1.mkv", "cat/lonely.mkv"} { + assert.Equal(t, database.PeernetSchemeFiles, f.row(ctx, p).ReleaseScheme) + } +} + +// Two nodes that imported the same NZB and kept the same files derive the same +// ID, whatever their local paths and folder names are. +func TestDerive_StableAcrossNodes(t *testing.T) { + ctx := context.Background() + f1, f2 := newFixture(t), newFixture(t) + f1.old(ctx, "movies/Name/x.mkv", srcA, "p1", "p2") + f1.old(ctx, "movies/Name/y.nfo", srcA, "q1") + f2.old(ctx, "other/Renamed/y.nfo", "/elsewhere/.nzbs/77/z.nzb.gz", "q1") + f2.old(ctx, "other/Renamed/x.mkv", "/elsewhere/.nzbs/77/z.nzb.gz", "p1", "p2") + indexAll(ctx, t, f1) + indexAll(ctx, t, f2) + assert.Equal(t, f1.row(ctx, "movies/Name/x.mkv").ReleaseID, f2.row(ctx, "other/Renamed/x.mkv").ReleaseID) + assert.NotEmpty(t, f1.row(ctx, "movies/Name/x.mkv").ReleaseID) + + // A node that kept only a subset derives a different release ID, but the + // same File IDs. + f3 := newFixture(t) + f3.old(ctx, "m/x.mkv", srcB, "p1", "p2") + indexAll(ctx, t, f3) + assert.NotEqual(t, f1.row(ctx, "movies/Name/x.mkv").ReleaseID, f3.row(ctx, "m/x.mkv").ReleaseID) + assert.Equal(t, f1.row(ctx, "movies/Name/x.mkv").FileID, f3.row(ctx, "m/x.mkv").FileID) +} + +func TestDerive_NoSourceMeansNoID(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + f.old(ctx, "cat/a.mkv", "", "a1") + f.old(ctx, "cat/b.mkv", " ", "b1") + res := indexAll(ctx, t, f) + assert.Equal(t, 0, res.Groups) + assert.Empty(t, f.row(ctx, "cat/a.mkv").ReleaseID) + assert.Empty(t, f.row(ctx, "cat/a.mkv").GroupKey) +} + +// A store-based ID always wins; a file never has both. +func TestDerive_StoreBasedWinsAndMigratedStoreIsDerived(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + nzb := f.release(ctx, "good", []string{"g1"})[0] + m, err := f.ms.ReadFileMetadata(nzb) + require.NoError(t, err) + m.SourceNzbPath = srcA + require.NoError(t, f.ms.WriteFileMetadata(nzb, m)) + f.old(ctx, "good/inline.mkv", srcA, "g2") // an old-format sibling of the same import + + // A store the metadata migration rebuilt from the surviving files. + mig := f.release(ctx, "_migrated/rel", []string{"m1"}, []string{"m2"}) + for _, p := range mig { + mm, err := f.ms.ReadFileMetadata(p) + require.NoError(t, err) + mm.SourceNzbPath = srcB + require.NoError(t, f.ms.WriteFileMetadata(p, mm)) + } + indexAll(ctx, t, f) + + got := f.row(ctx, nzb) + assert.Equal(t, database.PeernetSchemeNZB, got.ReleaseScheme) + st, err := f.ms.Store().ReadStore(m.StoreRef) + require.NoError(t, err) + want, err := ReleaseID(st) + require.NoError(t, err) + assert.Equal(t, want.String(), got.ReleaseID) + // The derived group is the old-format files only; it does not absorb the nzb-scheme file. + assert.Equal(t, f.wantFilesID("good/inline.mkv"), f.row(ctx, "good/inline.mkv").ReleaseID) + + for _, p := range mig { + r := f.row(ctx, p) + assert.Equal(t, database.PeernetSchemeFiles, r.ReleaseScheme, "a rebuilt store must not give an nzb-scheme ID") + assert.Equal(t, f.wantFilesID(mig...), r.ReleaseID) + } +} + +func TestDerive_GroupOverCapGetsNoID(t *testing.T) { + ctx := context.Background() + f := newFixture(t, WithMaxGroupFiles(3)) + for i := range 4 { + f.old(ctx, fmt.Sprintf("big/f%d.mkv", i), srcA, fmt.Sprintf("big%d", i)) + } + for i := range 3 { + f.old(ctx, fmt.Sprintf("ok/f%d.mkv", i), srcB, fmt.Sprintf("ok%d", i)) + } + res := indexAll(ctx, t, f) + assert.Equal(t, 4, res.OversizeFiles) + assert.Equal(t, 4, f.idx.OversizeFiles()) + assert.Empty(t, f.row(ctx, "big/f0.mkv").ReleaseID) + assert.NotEmpty(t, f.row(ctx, "ok/f0.mkv").ReleaseID, "exactly at the cap is allowed") +} + +// The default cap is the one release information can serve: a group above it +// must not be given an ID that publishing would then skip. +func TestDerive_DefaultCapIsTheReleaseInfoCap(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + const over = 2001 + for i := range over { + f.old(ctx, fmt.Sprintf("big/f%d.mkv", i), srcA, fmt.Sprintf("big%d", i)) + } + res := indexAll(ctx, t, f) + assert.Equal(t, over, res.OversizeFiles) + assert.Empty(t, f.row(ctx, "big/f0.mkv").ReleaseID) +} + +// A group that grew past the cap loses its ID, and the old ID is withdrawn. +func TestDerive_GroupGrowsPastCapLosesID(t *testing.T) { + ctx := context.Background() + var o orphans + f := newFixture(t, WithMaxGroupFiles(2), WithOrphanHandler(o.handle)) + f.old(ctx, "g/a.mkv", srcA, "a1") + f.old(ctx, "g/b.mkv", srcA, "b1") + indexAll(ctx, t, f) + old := f.row(ctx, "g/a.mkv").ReleaseID + require.NotEmpty(t, old) + + f.old(ctx, "g/c.mkv", srcA, "c1") + require.NoError(t, f.idx.IndexPath(ctx, "g/c.mkv")) + _, err := f.idx.DeriveDirty(ctx) + require.NoError(t, err) + assert.Empty(t, f.row(ctx, "g/a.mkv").ReleaseID) + assert.Empty(t, f.row(ctx, "g/c.mkv").ReleaseID) + assert.Equal(t, []string{old}, o.list()) +} + +func TestDerive_AddingAFileChangesIDAndWithdrawsOld(t *testing.T) { + ctx := context.Background() + var o orphans + f := newFixture(t, WithOrphanHandler(o.handle)) + f.old(ctx, "g/a.mkv", srcA, "a1") + indexAll(ctx, t, f) + before := f.row(ctx, "g/a.mkv").ReleaseID + + f.old(ctx, "g/b.mkv", srcA, "b1") + require.NoError(t, f.idx.IndexPath(ctx, "g/b.mkv")) + assert.True(t, f.idx.HasDirty()) + _, err := f.idx.DeriveDirty(ctx) + require.NoError(t, err) + assert.False(t, f.idx.HasDirty()) + + after := f.row(ctx, "g/a.mkv").ReleaseID + assert.NotEqual(t, before, after) + assert.Equal(t, after, f.row(ctx, "g/b.mkv").ReleaseID) + assert.Equal(t, f.wantFilesID("g/a.mkv", "g/b.mkv"), after) + assert.Equal(t, []string{before}, o.list(), "the superseded ID is reported so its records can be withdrawn") +} + +func TestDerive_RemovingFilesChangesIDAndLastFileWithdraws(t *testing.T) { + ctx := context.Background() + var o orphans + f := newFixture(t, WithOrphanHandler(o.handle)) + f.ms.SetFileObserver(NewObserver(f.idx)) + f.old(ctx, "g/a.mkv", srcA, "a1") + f.old(ctx, "g/b.mkv", srcA, "b1") + indexAll(ctx, t, f) + both := f.row(ctx, "g/a.mkv").ReleaseID + + require.NoError(t, f.ms.DeleteFileMetadata(ctx, "g/b.mkv")) + _, err := f.idx.DeriveDirty(ctx) + require.NoError(t, err) + one := f.row(ctx, "g/a.mkv").ReleaseID + assert.NotEqual(t, both, one) + assert.Equal(t, f.wantFilesID("g/a.mkv"), one) + assert.Equal(t, []string{both}, o.list()) + + require.NoError(t, f.ms.DeleteDirectory("g")) + _, err = f.idx.DeriveDirty(ctx) + require.NoError(t, err) + assert.ElementsMatch(t, []string{both, one}, o.list(), "deleting the last file withdraws its ID too") +} + +func TestDerive_RenameKeepsIDAndWithdrawsNothing(t *testing.T) { + ctx := context.Background() + var o orphans + f := newFixture(t, WithOrphanHandler(o.handle)) + f.ms.SetFileObserver(NewObserver(f.idx)) + f.old(ctx, "old/a.mkv", srcA, "a1") + f.old(ctx, "old/b.mkv", srcA, "b1") + indexAll(ctx, t, f) + before := f.row(ctx, "old/a.mkv").ReleaseID + require.NoError(t, f.ms.RenameDirectory("old", "new")) + _, err := f.idx.DeriveDirty(ctx) + require.NoError(t, err) + _, err = f.idx.Reconcile(ctx) + require.NoError(t, err) + _, err = f.idx.DeriveAll(ctx) + require.NoError(t, err) + assert.Equal(t, before, f.row(ctx, "new/a.mkv").ReleaseID) + assert.Empty(t, o.list()) +} + +// A metadata rewrite that changes nothing about identity (status, holes, a +// health check) must not touch the row, so a derived ID never flickers and the +// write path never marks a group dirty for nothing. +func TestDerive_IdentityNeutralRewriteDoesNotChurn(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + f.old(ctx, "g/a.mkv", srcA, "a1") + indexAll(ctx, t, f) + before := f.row(ctx, "g/a.mkv") + require.NoError(t, f.ms.UpdateFileStatus("g/a.mkv", metapb.FileStatus_FILE_STATUS_CORRUPTED)) + require.NoError(t, f.idx.IndexPath(ctx, "g/a.mkv")) + assert.False(t, f.idx.HasDirty()) + assert.Equal(t, before, f.row(ctx, "g/a.mkv")) +} + +func TestDerive_FileGainsStoreBecomesNZBAndOldIDWithdrawn(t *testing.T) { + ctx := context.Background() + var o orphans + f := newFixture(t, WithOrphanHandler(o.handle)) + f.old(ctx, "g/a.mkv", srcA, "a1") + indexAll(ctx, t, f) + derived := f.row(ctx, "g/a.mkv").ReleaseID + + p := f.release(ctx, "g2", []string{"a1"})[0] // same segments, now with a store + m, err := f.ms.ReadFileMetadata(p) + require.NoError(t, err) + m.SourceNzbPath = srcA + require.NoError(t, f.ms.WriteFileMetadata("g/a.mkv", m)) + require.NoError(t, f.idx.IndexPath(ctx, "g/a.mkv")) + _, err = f.idx.DeriveDirty(ctx) + require.NoError(t, err) + got := f.row(ctx, "g/a.mkv") + assert.Equal(t, database.PeernetSchemeNZB, got.ReleaseScheme) + assert.NotEqual(t, derived, got.ReleaseID) + assert.Equal(t, []string{derived}, o.list()) +} + +// The pass is bounded (small batches), resumable and idempotent. +func TestDerive_BatchedCancellableResumable(t *testing.T) { + ctx := context.Background() + f := newFixture(t, WithBatchSize(2), WithDeriveBatch(3, 4)) + for g := range 10 { + for k := range 3 { + f.old(ctx, fmt.Sprintf("c/g%d/f%d.mkv", g, k), fmt.Sprintf("/d/.nzbs/%d/x.nzb.gz", g), fmt.Sprintf("s%d_%d", g, k)) + } + } + _, err := f.idx.Backfill(ctx) + require.NoError(t, err) + + cctx, cancel := context.WithCancel(ctx) + cancel() + _, err = f.idx.DeriveAll(cctx) + require.ErrorIs(t, err, context.Canceled) + + res, err := f.idx.DeriveAll(ctx) + require.NoError(t, err) + assert.Equal(t, 10, res.Groups) + assert.Equal(t, 30, res.Assigned) + res, err = f.idx.DeriveAll(ctx) + require.NoError(t, err) + assert.Equal(t, 0, res.Assigned, "second pass changes nothing") + for g := range 10 { + var paths []string + for k := range 3 { + paths = append(paths, fmt.Sprintf("c/g%d/f%d.mkv", g, k)) + } + assert.Equal(t, f.wantFilesID(paths...), f.row(ctx, paths[0]).ReleaseID) + } +} + +// Reconcile must leave derived rows alone and must upgrade rows written +// before the group key existed. +func TestReconcile_KeepsDerivedAndFillsGroupKeys(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + f.old(ctx, "g/a.mkv", srcA, "a1") + f.old(ctx, "g/b.mkv", srcA, "b1") + indexAll(ctx, t, f) + derived := f.row(ctx, "g/a.mkv").ReleaseID + res, err := f.idx.Reconcile(ctx) + require.NoError(t, err) + assert.Equal(t, 0, res.Reindexed) + assert.Equal(t, derived, f.row(ctx, "g/a.mkv").ReleaseID) + + // A row as the previous version left it: file ID only, no group key. + e := f.row(ctx, "g/a.mkv") + require.NoError(t, f.repo.Upsert(ctx, database.PeernetContentEntry{FilePath: e.FilePath, FileID: e.FileID})) + require.NoError(t, f.repo.Upsert(ctx, database.PeernetContentEntry{FilePath: "g/b.mkv", FileID: f.row(ctx, "g/b.mkv").FileID})) + res, err = f.idx.Reconcile(ctx) + require.NoError(t, err) + assert.Equal(t, 2, res.Reindexed) + _, err = f.idx.DeriveAll(ctx) + require.NoError(t, err) + assert.Equal(t, derived, f.row(ctx, "g/a.mkv").ReleaseID, "same files, same ID") +} + +func TestDerive_ObserverEventsMarkOnlyTheirGroups(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + obs := NewObserver(f.idx) + f.ms.SetFileObserver(obs) + f.old(ctx, "g/a.mkv", srcA, "a1") + f.old(ctx, "h/b.mkv", srcB, "b1") + obs.FileWritten(ctx, "g/a.mkv") + obs.FileWritten(ctx, "h/b.mkv") + res, err := f.idx.DeriveDirty(ctx) + require.NoError(t, err) + assert.Equal(t, 2, res.Groups) + assert.Equal(t, 2, res.Assigned) + res, err = f.idx.DeriveDirty(ctx) + require.NoError(t, err) + assert.Equal(t, 0, res.Groups, "nothing dirty") +} + +func TestDerive_DirtyOverflowFallsBackToFullPass(t *testing.T) { + ctx := context.Background() + f := newFixture(t, WithMaxDirty(2)) + for g := range 4 { + f.old(ctx, fmt.Sprintf("d/g%d.mkv", g), fmt.Sprintf("/d/.nzbs/%d/x.nzb.gz", g), fmt.Sprintf("z%d", g)) + require.NoError(t, f.idx.IndexPath(ctx, fmt.Sprintf("d/g%d.mkv", g))) + } + res, err := f.idx.DeriveDirty(ctx) + require.NoError(t, err) + assert.Equal(t, 4, res.Assigned, "too many dirty groups: a full pass covers them all") +} + +// The same NZB imported into two folders gives two sets of rows with identical +// File IDs; the release is still the one file set. +func TestDerive_SameImportKeptUnderTwoFoldersIsOneRelease(t *testing.T) { + ctx := context.Background() + once, twice := newFixture(t), newFixture(t) + once.old(ctx, "c/One/x.mkv", srcA, "p1", "p2") + once.old(ctx, "c/One/y.nfo", srcA, "q1") + for _, dir := range []string{"c/One", "c/Two"} { + twice.old(ctx, dir+"/x.mkv", srcA, "p1", "p2") + twice.old(ctx, dir+"/y.nfo", srcA, "q1") + } + indexAll(ctx, t, once) + indexAll(ctx, t, twice) + want := once.row(ctx, "c/One/x.mkv").ReleaseID + require.NotEmpty(t, want) + assert.Equal(t, want, twice.row(ctx, "c/One/x.mkv").ReleaseID) + assert.Equal(t, want, twice.row(ctx, "c/Two/y.nfo").ReleaseID) +} + +// A cancelled or failed full pass must not lose the work it took: the next +// DeriveDirty still has it. +func TestDerive_FullPassCancelledKeepsItsWork(t *testing.T) { + ctx := context.Background() + f := newFixture(t, WithMaxDirty(2)) + for g := range 4 { + f.old(ctx, fmt.Sprintf("d/g%d.mkv", g), fmt.Sprintf("/d/.nzbs/%d/x.nzb.gz", g), fmt.Sprintf("z%d", g)) + require.NoError(t, f.idx.IndexPath(ctx, fmt.Sprintf("d/g%d.mkv", g))) + } + cctx, cancel := context.WithCancel(ctx) + cancel() + _, err := f.idx.DeriveDirty(cctx) + require.ErrorIs(t, err, context.Canceled) + assert.True(t, f.idx.HasDirty(), "the full pass must be remembered after a cancel") + res, err := f.idx.DeriveDirty(ctx) + require.NoError(t, err) + assert.Equal(t, 4, res.Assigned) +} + +func TestDerive_DeriveAllCancelledKeepsDirtyWork(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + f.old(ctx, "d/g.mkv", "/d/.nzbs/1/x.nzb.gz", "z0") + require.NoError(t, f.idx.IndexPath(ctx, "d/g.mkv")) + require.True(t, f.idx.HasDirty()) + cctx, cancel := context.WithCancel(ctx) + cancel() + _, err := f.idx.DeriveAll(cctx) + require.ErrorIs(t, err, context.Canceled) + assert.True(t, f.idx.HasDirty(), "dirty groups taken by a cancelled DeriveAll must come back") + res, err := f.idx.DeriveDirty(ctx) + require.NoError(t, err) + assert.Equal(t, 1, res.Assigned) +} diff --git a/internal/peernet/contentid/doc.go b/internal/peernet/contentid/doc.go index a853dde5b..29619d624 100644 --- a/internal/peernet/contentid/doc.go +++ b/internal/peernet/contentid/doc.go @@ -38,6 +38,33 @@ // hashed exactly as if the shared source's segments had been inlined, so the // ID does not depend on how the metadata happens to be stored. // +// # Library-derived Release ID ("files" scheme) +// +// A file imported before the NZB store existed (segments kept inline per file) +// has a File ID but no NZB segment list, so no Release ID of the form above. +// Such a release is identified by its files instead: +// +// files release ID = SHA-256( domain("altmount-peernet/release-files/v1") +// +// || fileID(1) || fileID(2) || ... || fileID(n) ) +// +// where domain() is the same u32-length-prefixed label as above, the file IDs +// are the 32-byte File IDs of the release's files sorted ascending by plain +// bytewise comparison with duplicates removed (a file kept under two folders is +// one file), and there is no count and no separator (every item has a fixed +// width). n, the number of distinct File IDs, is at least 1 and at most +// 10,000 (a node only derives +// IDs for groups up to MaxDerivedReleaseFiles, 2000). Records that carry such an ID say so in +// their release_scheme field (2); the two ID spaces are domain-separated, so +// an "nzb" ID and a "files" ID never collide. +// +// Which files make up "the release" is a local decision (see the Indexer): the +// indexed files that came from one import, found through the stored source +// NZB path of their metadata. Two nodes that imported the same NZB and kept +// the same file set derive the same ID; different kept subsets give different +// IDs. File IDs match across both schemes, so health evidence is shared per +// file. +// // Every variable-length item is length-prefixed and every list is // count-prefixed, and the domain label differs per ID kind, so two different // structures cannot produce the same input stream. Future layouts must use a diff --git a/internal/peernet/contentid/files_id.go b/internal/peernet/contentid/files_id.go new file mode 100644 index 000000000..03d4c9c33 --- /dev/null +++ b/internal/peernet/contentid/files_id.go @@ -0,0 +1,64 @@ +package contentid + +import ( + "bytes" + "errors" + "fmt" + "sort" +) + +// MaxDerivedReleaseFiles is the most files one library-derived release may +// have on this node. A larger group is not given a Release ID. It is the single +// local limit for deriving an ID and for describing the release when +// publishing (release information reads the metadata of every file, so it must +// stay bounded). +const MaxDerivedReleaseFiles = 2000 + +// maxIDFiles is the wire bound of the files Release ID function (the most +// files a bundle may list); it is independent of the local cap above. +const maxIDFiles = 10000 + +const releaseFilesDomain = "altmount-peernet/release-files/v1" + +var ( + // ErrNoFiles means a library-derived Release ID was asked for with no files. + ErrNoFiles = errors.New("contentid: no files") + // ErrTooManyFiles means the group is over the 10,000-file wire bound. + ErrTooManyFiles = errors.New("contentid: too many files") +) + +// ReleaseIDFromFiles computes the library-derived ("files" scheme) Release ID +// of a release whose NZB segment list is not available: SHA-256 over the +// domain label (u32 length prefix, as in every other ID) followed by the +// distinct File IDs of the release's files sorted ascending bytewise, each 32 +// bytes, with no count and no separators. See the package documentation. A +// File ID listed more than once (the same file kept under two folders) counts +// once. The input is not modified. A zero File ID, an empty list or more than +// 10,000 distinct files is an error. +func ReleaseIDFromFiles(files []ID) (ID, error) { + if len(files) == 0 { + return ID{}, ErrNoFiles + } + sorted := make([]ID, len(files)) + copy(sorted, files) + for _, f := range sorted { + if f.IsZero() { + return ID{}, fmt.Errorf("%w: zero file id", ErrInvalidID) + } + } + sort.Slice(sorted, func(a, b int) bool { return bytes.Compare(sorted[a][:], sorted[b][:]) < 0 }) + distinct := sorted[:1] + for _, f := range sorted[1:] { + if f != distinct[len(distinct)-1] { + distinct = append(distinct, f) + } + } + if len(distinct) > maxIDFiles { + return ID{}, fmt.Errorf("%w: %d", ErrTooManyFiles, len(distinct)) + } + w := newWriter(releaseFilesDomain) + for _, f := range distinct { + _, _ = w.h.Write(f[:]) + } + return w.sum(), nil +} diff --git a/internal/peernet/contentid/files_id_test.go b/internal/peernet/contentid/files_id_test.go new file mode 100644 index 000000000..f942a3c16 --- /dev/null +++ b/internal/peernet/contentid/files_id_test.go @@ -0,0 +1,103 @@ +package contentid + +import ( + "crypto/sha256" + "encoding/binary" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func fid(b byte) ID { + var id ID + for i := range id { + id[i] = b + } + return id +} + +// referenceFilesID is an independent spelling of the documented byte layout. +func referenceFilesID(sorted ...ID) ID { + h := sha256.New() + var l [4]byte + binary.BigEndian.PutUint32(l[:], uint32(len("altmount-peernet/release-files/v1"))) + h.Write(l[:]) + h.Write([]byte("altmount-peernet/release-files/v1")) + for _, id := range sorted { + h.Write(id[:]) + } + var out ID + copy(out[:], h.Sum(nil)) + return out +} + +func TestReleaseIDFromFiles_Layout(t *testing.T) { + got, err := ReleaseIDFromFiles([]ID{fid(3), fid(1), fid(2)}) + require.NoError(t, err) + assert.Equal(t, referenceFilesID(fid(1), fid(2), fid(3)), got) +} + +func TestReleaseIDFromFiles_OrderIndependentAndInputKept(t *testing.T) { + in := []ID{fid(9), fid(4)} + a, err := ReleaseIDFromFiles(in) + require.NoError(t, err) + b, err := ReleaseIDFromFiles([]ID{fid(4), fid(9)}) + require.NoError(t, err) + assert.Equal(t, a, b) + assert.Equal(t, []ID{fid(9), fid(4)}, in, "input slice must not be reordered") +} + +func TestReleaseIDFromFiles_DistinctFromOtherKinds(t *testing.T) { + one, err := ReleaseIDFromFiles([]ID{fid(5)}) + require.NoError(t, err) + assert.NotEqual(t, fid(5), one) + two, err := ReleaseIDFromFiles([]ID{fid(5), fid(6)}) + require.NoError(t, err) + assert.NotEqual(t, one, two) +} + +func TestReleaseIDFromFiles_Rejects(t *testing.T) { + _, err := ReleaseIDFromFiles(nil) + assert.ErrorIs(t, err, ErrNoFiles) + _, err = ReleaseIDFromFiles([]ID{fid(1), {}}) + assert.ErrorIs(t, err, ErrInvalidID) + over := make([]ID, maxIDFiles+1) + for i := range over { + binary.BigEndian.PutUint32(over[i][:4], uint32(i+1)) + } + _, err = ReleaseIDFromFiles(over) + assert.ErrorIs(t, err, ErrTooManyFiles) +} + +func TestReleaseIDFromFiles_CapBoundaryAccepted(t *testing.T) { + ids := make([]ID, maxIDFiles) + for i := range ids { + binary.BigEndian.PutUint32(ids[i][:4], uint32(i+1)) + } + _, err := ReleaseIDFromFiles(ids) + assert.NoError(t, err) +} + +// A file kept under two folders (the same NZB imported twice) is one file of +// the release: the ID is over the distinct File IDs, so a node holding the +// duplicate derives the same ID as one that does not. +func TestReleaseIDFromFiles_DuplicatesAreOneFile(t *testing.T) { + once, err := ReleaseIDFromFiles([]ID{fid(1), fid(2)}) + require.NoError(t, err) + twice, err := ReleaseIDFromFiles([]ID{fid(2), fid(1), fid(2), fid(1), fid(1)}) + require.NoError(t, err) + assert.Equal(t, once, twice) + assert.Equal(t, referenceFilesID(fid(1), fid(2)), twice) +} + +func TestReleaseIDFromFiles_CapCountsDistinctFiles(t *testing.T) { + ids := make([]ID, 0, maxIDFiles*2) + for i := range maxIDFiles { + var id ID + binary.BigEndian.PutUint32(id[:4], uint32(i+1)) + ids = append(ids, id, id) + } + _, err := ReleaseIDFromFiles(ids) + assert.NoError(t, err, "10,000 distinct files listed twice is still 10,000 files") +} diff --git a/internal/peernet/contentid/indexer.go b/internal/peernet/contentid/indexer.go index 713f8a1a9..3a324cf51 100644 --- a/internal/peernet/contentid/indexer.go +++ b/internal/peernet/contentid/indexer.go @@ -2,6 +2,8 @@ package contentid import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "io/fs" @@ -40,6 +42,15 @@ type Repository interface { DeleteByPrefix(ctx context.Context, dir string) error RenamePrefix(ctx context.Context, oldDir, newDir string) error ListPage(ctx context.Context, after string, limit int) ([]database.PeernetContentEntry, error) + GetByPath(ctx context.Context, filePath string) (database.PeernetContentEntry, bool, error) + RefByPath(ctx context.Context, filePath string) (database.PeernetContentRef, bool, error) + RefsByPrefix(ctx context.Context, dir string, limit int) ([]database.PeernetContentRef, error) + GroupSummaries(ctx context.Context, after string, limit int) ([]database.PeernetGroupSummary, error) + GroupCounts(ctx context.Context, keys []string) (map[string]int, error) + GroupMembers(ctx context.Context, keys []string) ([]database.PeernetContentEntry, error) + ApplyDerived(ctx context.Context, updates []database.PeernetDerivedUpdate) (int, error) + ClearDerivedGroup(ctx context.Context, key string, batch int) ([]string, error) + StillIndexed(ctx context.Context, releaseIDs []string) (map[string]bool, error) } // ServiceSource adapts a metadata.MetadataService to Source. @@ -89,6 +100,7 @@ type Indexer struct { repo Repository batchSize int progress func(BackfillResult) + derive *deriveState // Files of one release share one store; hashing a large store once per // file would be quadratic, so the most recent result is remembered. It is @@ -100,7 +112,7 @@ type Indexer struct { // NewIndexer builds an Indexer. func NewIndexer(src Source, repo Repository, opts ...Option) *Indexer { - i := &Indexer{src: src, repo: repo, batchSize: defaultBatchSize} + i := &Indexer{src: src, repo: repo, batchSize: defaultBatchSize, derive: newDeriveState()} for _, o := range opts { o(i) } @@ -142,9 +154,24 @@ func migratedStore(ref string) bool { return false } +// groupKey is the local-only key shared by the files of one import: a hash of +// the source NZB path the importer recorded in each file's metadata (it holds +// the queue item's own folder, so it is unique per import). It is empty when +// the metadata names no source. It is never transmitted and never part of an +// ID: only the File IDs of the group are. +func groupKey(sourceNzbPath string) string { + p := strings.TrimSpace(strings.ReplaceAll(sourceNzbPath, "\\", "/")) + if p == "" { + return "" + } + sum := sha256.Sum256([]byte("altmount-peernet/import-group/v1\x00" + p)) + return hex.EncodeToString(sum[:]) +} + // Entry computes the index row for a virtual path without storing it. The -// release ID is empty when the metadata has no shared store, or only one the -// migration rebuilt from the surviving files. +// release ID is empty (scheme none) when the metadata has no shared store, or +// only one the migration rebuilt from the surviving files; such files get a +// library-derived ID later, from their group (see DeriveAll). func (i *Indexer) Entry(ctx context.Context, virtualPath string) (database.PeernetContentEntry, error) { var zero database.PeernetContentEntry if err := ctx.Err(); err != nil { @@ -161,7 +188,7 @@ func (i *Indexer) Entry(ctx context.Context, virtualPath string) (database.Peern if err != nil { return zero, fmt.Errorf("file id %q: %w", virtualPath, err) } - e := database.PeernetContentEntry{FilePath: virtualPath, FileID: fileID.String()} + e := database.PeernetContentEntry{FilePath: virtualPath, FileID: fileID.String(), GroupKey: groupKey(m.SourceNzbPath)} if m.StoreRef != "" && !migratedStore(m.StoreRef) { store, err := i.src.ReadStore(m.StoreRef) if err != nil { @@ -169,7 +196,7 @@ func (i *Indexer) Entry(ctx context.Context, virtualPath string) (database.Peern } switch rid, err := i.releaseID(store); { case err == nil: - e.ReleaseID = rid.String() + e.ReleaseID, e.ReleaseScheme = rid.String(), database.PeernetSchemeNZB case errors.Is(err, ErrNoSegments): // A store with no real article cannot name a release; keep the file ID. default: @@ -179,11 +206,49 @@ func (i *Indexer) Entry(ctx context.Context, virtualPath string) (database.Peern return e, nil } -// IndexPath computes and stores the IDs of one virtual file. +// sameIdentity reports whether a stored row already says what a freshly +// computed entry says. A derived ID (scheme files) is not part of the entry: +// it belongs to the row's group, so a row that has one is the same as an entry +// without a store-based ID as long as the file and the group are unchanged. +func sameIdentity(row, e database.PeernetContentEntry) bool { + if row.FileID != e.FileID || row.GroupKey != e.GroupKey { + return false + } + if e.ReleaseScheme == database.PeernetSchemeNZB { + return row.ReleaseScheme == database.PeernetSchemeNZB && row.ReleaseID == e.ReleaseID + } + return row.ReleaseScheme != database.PeernetSchemeNZB +} + +// IndexPath computes and stores the IDs of one virtual file. A file whose row +// already says the same is not written again (health checks and status +// updates rewrite metadata all the time), so the row never flickers and no +// group is re-derived for nothing. A changed row marks its group, and the old +// one if the group moved, for the next derive pass. func (i *Indexer) IndexPath(ctx context.Context, virtualPath string) error { e, err := i.Entry(ctx, virtualPath) if err != nil { return err } - return i.repo.Upsert(ctx, e) + old, had, err := i.repo.GetByPath(ctx, virtualPath) + if err != nil { + return fmt.Errorf("read index row %q: %w", virtualPath, err) + } + if had && sameIdentity(old, e) { + return nil + } + if had && keepsBundleGrouping(old, e) { + // The release identity came from a verified bundle; the metadata of a + // bundle import cannot recompute it or its group. + return nil + } + if err := i.repo.Upsert(ctx, e); err != nil { + return err + } + if had { + i.noteChanged(old, e) + } else { + i.noteChanged(database.PeernetContentEntry{}, e) + } + return nil } diff --git a/internal/peernet/contentid/indexer_test.go b/internal/peernet/contentid/indexer_test.go index bee815b44..07eb3ced2 100644 --- a/internal/peernet/contentid/indexer_test.go +++ b/internal/peernet/contentid/indexer_test.go @@ -174,7 +174,7 @@ func TestObserver_FailureNeverFailsTheWrite(t *testing.T) { require.NoError(t, f.ms.DeleteDirectory("rel")) } -type failingRepo struct{} +type failingRepo struct{ Repository } var errRepo = fmt.Errorf("repo down") @@ -186,6 +186,15 @@ func (failingRepo) MissingPaths(context.Context, []string) ([]string, error) { r func (failingRepo) Delete(context.Context, string) error { return errRepo } func (failingRepo) DeleteByPrefix(context.Context, string) error { return errRepo } func (failingRepo) RenamePrefix(context.Context, string, string) error { return errRepo } +func (failingRepo) GetByPath(context.Context, string) (database.PeernetContentEntry, bool, error) { + return database.PeernetContentEntry{}, false, errRepo +} +func (failingRepo) RefByPath(context.Context, string) (database.PeernetContentRef, bool, error) { + return database.PeernetContentRef{}, false, errRepo +} +func (failingRepo) RefsByPrefix(context.Context, string, int) ([]database.PeernetContentRef, error) { + return nil, errRepo +} func (failingRepo) ListPage(context.Context, string, int) ([]database.PeernetContentEntry, error) { return nil, errRepo } diff --git a/internal/peernet/contentid/observer.go b/internal/peernet/contentid/observer.go index e247aae83..136fcdb9c 100644 --- a/internal/peernet/contentid/observer.go +++ b/internal/peernet/contentid/observer.go @@ -64,15 +64,28 @@ func (o *Observer) FileWritten(ctx context.Context, virtualPath string) { // FileDeleted drops the index row of a removed or moved file. func (o *Observer) FileDeleted(ctx context.Context, virtualPath string) { + ref, had, _ := o.idx.repo.RefByPath(ctx, virtualPath) if err := o.idx.repo.Delete(ctx, virtualPath); err != nil && !errors.Is(err, context.Canceled) { o.logs.warn(ctx, "Peernet content index delete failed", virtualPath, err) + return + } + if had { + o.idx.noteRemoved(ref) } } // DirectoryDeleted drops every index row beneath a removed directory. func (o *Observer) DirectoryDeleted(ctx context.Context, virtualPath string) { + refs, _ := o.idx.repo.RefsByPrefix(ctx, virtualPath, maxDirRefs) if err := o.idx.repo.DeleteByPrefix(ctx, virtualPath); err != nil && !errors.Is(err, context.Canceled) { o.logs.warn(ctx, "Peernet content index directory delete failed", virtualPath, err) + return + } + if len(refs) >= maxDirRefs { + o.idx.markAllDirty() // too many to track one by one + } + for _, ref := range refs { + o.idx.noteRemoved(ref) } } diff --git a/internal/peernet/contentid/reconcile.go b/internal/peernet/contentid/reconcile.go index ef3b21533..abdb23562 100644 --- a/internal/peernet/contentid/reconcile.go +++ b/internal/peernet/contentid/reconcile.go @@ -53,16 +53,17 @@ func (i *Indexer) Reconcile(ctx context.Context) (ReconcileResult, error) { } after = page[len(page)-1].FilePath upserts := make([]database.PeernetContentEntry, 0, len(page)) + olds := make(map[string]database.PeernetContentEntry, len(page)) for _, row := range page { if err := ctx.Err(); err != nil { - i.flushReconcile(ctx, upserts, &res) + i.flushReconcile(ctx, upserts, olds, &res) return res, err } res.Checked++ act, entry, err := i.reconcileRow(ctx, row) if err != nil { if ctx.Err() != nil { - i.flushReconcile(ctx, upserts, &res) + i.flushReconcile(ctx, upserts, olds, &res) return res, ctx.Err() } return res, err @@ -76,7 +77,7 @@ func (i *Indexer) Reconcile(ctx context.Context) (ReconcileResult, error) { res.Skipped++ } } - if err := i.flushReconcile(ctx, upserts, &res); err != nil { + if err := i.flushReconcile(ctx, upserts, olds, &res); err != nil { return res, err } } @@ -87,7 +88,7 @@ func (i *Indexer) Reconcile(ctx context.Context) (ReconcileResult, error) { return res, nil } -func (i *Indexer) flushReconcile(ctx context.Context, upserts []database.PeernetContentEntry, res *ReconcileResult) error { +func (i *Indexer) flushReconcile(ctx context.Context, upserts []database.PeernetContentEntry, olds map[string]database.PeernetContentEntry, res *ReconcileResult) error { if len(upserts) == 0 { return nil } @@ -95,6 +96,9 @@ func (i *Indexer) flushReconcile(ctx context.Context, upserts []database.Peernet return fmt.Errorf("write reindexed rows: %w", err) } res.Reindexed += len(upserts) + for _, e := range upserts { + i.noteChanged(olds[e.FilePath], e) + } return nil } @@ -134,7 +138,16 @@ func (i *Indexer) reconcileRow(ctx context.Context, row database.PeernetContentE case err != nil: return actSkip, zero, nil } - if fid.String() == row.FileID && (m.StoreRef != "" && !migratedStore(m.StoreRef)) == (row.ReleaseID != "") { + if fid.String() == row.FileID && row.GroupKey != "" && groupKey(m.SourceNzbPath) == "" && + (m.StoreRef == "" || migratedStore(m.StoreRef)) { + return actKeep, zero, nil // grouped by a verified bundle: the metadata cannot recompute that + } + // A derived ID (scheme files) belongs to the row's group, not to the + // metadata: a row is unchanged while the file, its group and whether it + // has a store-based ID agree. + storeBased := m.StoreRef != "" && !migratedStore(m.StoreRef) + if fid.String() == row.FileID && groupKey(m.SourceNzbPath) == row.GroupKey && + storeBased == (row.ReleaseScheme == database.PeernetSchemeNZB) { return actKeep, zero, nil } entry, err := i.Entry(ctx, row.FilePath) @@ -144,7 +157,7 @@ func (i *Indexer) reconcileRow(ctx context.Context, row database.PeernetContentE } return actSkip, zero, nil } - if entry.FileID == row.FileID && entry.ReleaseID == row.ReleaseID { + if sameIdentity(row, entry) { return actKeep, zero, nil // e.g. a store with no real article keeps an empty release ID } return actReindex, entry, nil diff --git a/internal/peernet/hints/advice.go b/internal/peernet/hints/advice.go new file mode 100644 index 000000000..8f12991ae --- /dev/null +++ b/internal/peernet/hints/advice.go @@ -0,0 +1,219 @@ +package hints + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/peernet/netid" + "github.com/kipsilabs/altmount/internal/peernet/records" +) + +type adviceEvidence struct { + openHealthy int + privHealthy int + privNewest time.Time + privNet netid.NetworkID + openNet netid.NetworkID + unhealthyNewest time.Time +} + +// HealthAdvice returns what peer evidence allows for one file. Evidence is +// found by File ID alone, whatever release ID its publisher used: the same +// articles are the same file in every release they are part of, and a file +// imported before the NZB store existed has no NZB-based release ID at all. +// releaseID is therefore optional (empty when the file has none) and is not +// used for the lookup. localBackbones +// are the raw backbone labels of the local providers. now is the caller's +// current time (normally the injected clock). Errors from the store or the +// peer table are returned with AdviceNone: advice is never guessed. +func (a *Advisor) HealthAdvice(ctx context.Context, releaseID, fileID string, localBackbones []string, now time.Time) (Advice, error) { + if (releaseID != "" && !validID(releaseID)) || !validID(fileID) { + return Advice{}, errors.New("hints: release and file ids must be 64 lower-case hex characters") + } + local := normaliseSet(localBackbones) + if len(local) == 0 { + return Advice{}, nil + } + var ev adviceEvidence + for _, n := range a.receiving() { + if err := a.gather(ctx, n, fileID, local, now, &ev); err != nil { + return Advice{}, err + } + } + return a.decide(fileID, ev, now), nil +} + +func (a *Advisor) gather(ctx context.Context, n Network, fileID string, local map[string]struct{}, now time.Time, ev *adviceEvidence) error { + atts, err := a.cfg.Store.AttestationsForFiles(ctx, n.ID.String(), []string{fileID}, now.Add(-records.AttestationValidity)) + if err != nil { + return fmt.Errorf("hints: attestations: %w", err) + } + healthy := map[string]struct{}{} // distinct groups attesting healthy + unhealthyG := map[string]struct{}{} // distinct groups attesting unhealthy + seenHealthy := map[string]struct{}{} + known := newKnownCache(a, ctx, n, now) + if err := known.prefetch(a.relevantPublishers(atts, fileID, local, now)); err != nil { + return err + } + var newest, unhealthyNewest time.Time + for _, att := range atts { + if !strings.EqualFold(att.FileID, fileID) || !a.usable(att, now) { + continue + } + subset, overlap := relation(att.Backbones, local) + if !att.Healthy { + if !overlap { + continue + } + // The same standing as healthy evidence: on the open network a + // publisher known for MinKnown, so fresh identities cannot drain + // the re-check budget. + ok, err := known.eligible(att.PublisherNodeID) + if err != nil { + return err + } + if ok { + unhealthyG[known.groupOf(att.PublisherNodeID)] = struct{}{} + if att.CheckedAt.After(unhealthyNewest) { + unhealthyNewest = att.CheckedAt + } + } + continue + } + if !subset || !CoverageSufficient(att.SegmentsChecked, att.TotalSegments) { + continue + } + if _, dup := seenHealthy[att.PublisherNodeID]; dup { + if att.CheckedAt.After(newest) { + newest = att.CheckedAt + } + continue + } + ok, err := known.eligible(att.PublisherNodeID) + if err != nil { + return err + } + if ok { + seenHealthy[att.PublisherNodeID] = struct{}{} + healthy[known.groupOf(att.PublisherNodeID)] = struct{}{} + if att.CheckedAt.After(newest) { + newest = att.CheckedAt + } + } + } + // Unhealthy evidence needs the same number of distinct publishers as + // healthy corroboration, so a lone identity cannot veto or force work. + if len(unhealthyG) >= a.threshold(n) && unhealthyNewest.After(ev.unhealthyNewest) { + ev.unhealthyNewest = unhealthyNewest + } + if n.ID.IsOpen() { + ev.openHealthy += len(healthy) + ev.openNet = n.ID + } else if len(healthy) >= a.cfg.PrivateThreshold && newest.After(ev.privNewest) { + ev.privHealthy += len(healthy) + ev.privNewest, ev.privNet = newest, n.ID + } + return nil +} + +// relevantPublishers lists, in order and without repeats, the publishers +// whose attestations gather may count, so their standing can be fetched in +// one go. +func (a *Advisor) relevantPublishers(atts []database.PeernetAttestation, fileID string, local map[string]struct{}, now time.Time) []string { + var out []string + seen := map[string]struct{}{} + for _, att := range atts { + if !strings.EqualFold(att.FileID, fileID) || !a.usable(att, now) { + continue + } + subset, overlap := relation(att.Backbones, local) + if att.Healthy && (!subset || !CoverageSufficient(att.SegmentsChecked, att.TotalSegments)) || !att.Healthy && !overlap { + continue + } + if _, dup := seen[att.PublisherNodeID]; !dup { + seen[att.PublisherNodeID] = struct{}{} + out = append(out, att.PublisherNodeID) + } + } + return out +} + +func (a *Advisor) decide(fileKey string, ev adviceEvidence, now time.Time) Advice { + if !ev.unhealthyNewest.IsZero() { + // Unhealthy evidence, even next to healthy evidence, never shortens + // anything; at most it may move a check forward, once. Deciding is + // read-only: the budget is spent by SpendRecheck when a check really + // was moved. + if a.acted(fileKey, ev.unhealthyNewest) { + return Advice{} + } + return Advice{Kind: AdviceRecheckSooner, EvidenceAt: ev.unhealthyNewest} + } + if ev.privHealthy >= a.cfg.PrivateThreshold && !ev.privNewest.IsZero() { + if until := ev.privNewest.Add(records.AttestationValidity); until.After(now) { + return Advice{Kind: AdviceMaySkipOneCheck, SkipUntil: until, Network: ev.privNet} + } + } + if ev.openHealthy >= a.corroboration() { + return Advice{Kind: AdviceReduceSampleToFloor, Network: ev.openNet} + } + return Advice{} +} + +// acted reports whether evidence of the given check time (or newer) for the +// file was already acted on. +func (a *Advisor) acted(key string, newest time.Time) bool { + a.mu.Lock() + defer a.mu.Unlock() + prev, seen := a.advised[key] + return seen && !newest.After(prev) +} + +// markAdvised records acted-on evidence, evicting the entry with the oldest +// evidence when the memory is full. The caller holds a.mu. +func (a *Advisor) markAdvised(key string, newest time.Time) { + if _, have := a.advised[key]; !have && len(a.advised) >= maxAdvised { + var oldestKey string + var oldest time.Time + first := true + for k, t := range a.advised { + if first || t.Before(oldest) { + oldestKey, oldest, first = k, t, false + } + } + delete(a.advised, oldestKey) + } + a.advised[key] = newest +} + +// forget clears the memory of acted-on evidence. +func (a *Advisor) forget() { + a.mu.Lock() + a.advised = map[string]time.Time{} + a.mu.Unlock() +} + +// SpendRecheck pays one token of the shared budget for moving a check forward +// on the unhealthy evidence of the given check time, and remembers that the +// evidence was acted on. It reports false when this evidence was already acted +// on or no token is left. It is called only once a check really was moved: +// HealthAdvice itself spends nothing. +func (a *Advisor) SpendRecheck(_, fileID string, evidenceAt, now time.Time) bool { + // Evidence is about the file, so it is paid for once per file however + // many release IDs reach it. + key := fileID + a.mu.Lock() + defer a.mu.Unlock() + if prev, seen := a.advised[key]; seen && !evidenceAt.After(prev) { + return false + } + if !a.cfg.Budget.Take(now) { + return false + } + a.markAdvised(key, evidenceAt) + return true +} diff --git a/internal/peernet/hints/candidates.go b/internal/peernet/hints/candidates.go new file mode 100644 index 000000000..975efd856 --- /dev/null +++ b/internal/peernet/hints/candidates.go @@ -0,0 +1,137 @@ +package hints + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/peernet/netid" + "github.com/kipsilabs/altmount/internal/peernet/records" +) + +// Candidate is one release found for a media key. Title is peer-supplied data +// to be rendered as text only. +type Candidate struct { + ReleaseID string + Title string + TotalSize int64 + CreatedAt time.Time + // Networks lists every receiving network the release was seen in, sorted. + Networks []netid.NetworkID + // Own marks releases this node published itself. + Own bool + Health NetworkHealth +} + +func validIMDb(s string) bool { + if len(s) < 3 || len(s) > 2+maxIMDbDigits || s[:2] != "tt" { + return false + } + for i := 2; i < len(s); i++ { + if s[i] < '0' || s[i] > '9' { + return false + } + } + return true +} + +// Candidates returns the release records known for a media key across the +// networks this node receives from, one entry per release ID, healthy-labelled +// first, then newest, then by release ID; at most CandidateLimit entries. A +// nil season and episode match movies only (see the store contract). +func (a *Advisor) Candidates(ctx context.Context, imdbID string, season, episode *int) ([]Candidate, error) { + if !validIMDb(imdbID) { + return nil, errors.New("hints: invalid imdb id") + } + if (season != nil && *season < 0) || (episode != nil && *episode < 0) { + return nil, errors.New("hints: negative season or episode") + } + nets := a.receiving() + if len(nets) == 0 { + return nil, nil + } + idStrs := make([]string, len(nets)) + for i, n := range nets { + idStrs[i] = n.ID.String() + } + rows, err := a.cfg.Store.CandidatesByMedia(ctx, idStrs, imdbID, season, episode, a.cfg.CandidateLimit) + if err != nil { + return nil, fmt.Errorf("hints: candidates: %w", err) + } + merged := a.merge(rows) + if len(merged) == 0 { + return nil, nil + } + ids := make([]string, len(merged)) + for i, c := range merged { + ids[i] = c.ReleaseID + } + health, err := a.NetworkHealthy(ctx, ids...) + if err != nil { + return nil, err + } + for i := range merged { + merged[i].Health = health[merged[i].ReleaseID] + } + sort.Slice(merged, func(i, j int) bool { + x, y := merged[i], merged[j] + switch { + case x.Health.Healthy != y.Health.Healthy: + return x.Health.Healthy + case !x.CreatedAt.Equal(y.CreatedAt): + return x.CreatedAt.After(y.CreatedAt) + } + return x.ReleaseID < y.ReleaseID + }) + if len(merged) > a.cfg.CandidateLimit { + merged = merged[:a.cfg.CandidateLimit] + } + return merged, nil +} + +// merge de-duplicates rows by release ID; the newest record defines the entry. +func (a *Advisor) merge(rows []database.PeernetCandidate) []Candidate { + byID := map[string]*Candidate{} + var order []string + for _, r := range rows { + nw, err := netid.ParseNetworkID(r.NetworkID) + title := records.CleanTitle(r.Title) + if err != nil || !validID(r.ReleaseID) || title == "" { + continue + } + own := r.Source == database.PeernetRecordSourceLocal || strings.EqualFold(r.PublisherNodeID, a.cfg.SelfNodeID) + c := byID[r.ReleaseID] + if c == nil { + c = &Candidate{ReleaseID: r.ReleaseID} + byID[r.ReleaseID] = c + order = append(order, r.ReleaseID) + } + if c.Title == "" || r.CreatedAt.After(c.CreatedAt) { + c.Title, c.TotalSize, c.CreatedAt = title, r.TotalSize, r.CreatedAt + } + c.Own = c.Own || own + if !containsNet(c.Networks, nw) { + c.Networks = append(c.Networks, nw) + } + } + out := make([]Candidate, 0, len(order)) + for _, id := range order { + c := byID[id] + sort.Slice(c.Networks, func(i, j int) bool { return c.Networks[i].String() < c.Networks[j].String() }) + out = append(out, *c) + } + return out +} + +func containsNet(list []netid.NetworkID, n netid.NetworkID) bool { + for _, x := range list { + if x == n { + return true + } + } + return false +} diff --git a/internal/peernet/hints/files_test.go b/internal/peernet/hints/files_test.go new file mode 100644 index 000000000..1a6f445ac --- /dev/null +++ b/internal/peernet/hints/files_test.go @@ -0,0 +1,345 @@ +package hints + +import ( + "context" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/peernet/netid" +) + +var ( + // The same file as published by an instance that kept the NZB ("nzb" + // release ID) and by one that only knows the files ("files" release ID). + relNZB = hexID(0x11) + relFiles = hexID(0x12) + fileA = hexID(0x21) + fileB = hexID(0x22) + fileC = hexID(0x23) +) + +// seedFileEv adds node i's attestation of fileID under release rel. +func (e *env) seedFileEv(nw netid.NetworkID, rel, fileID string, i int, mut func(*database.PeernetAttestation)) { + a := database.PeernetAttestation{ + PublisherNodeID: node(i), FileID: fileID, Healthy: true, Backbones: []string{"omicron"}, + CheckedAt: e.now.Add(-time.Hour), CreatedAt: e.now.Add(-time.Hour), Source: database.PeernetRecordSourcePeer, + SegmentsChecked: 100, TotalSegments: 1000, + } + if mut != nil { + mut(&a) + } + e.store.add(nw, rel, a) + e.peers.first[node(i)] = e.now.Add(-7 * 24 * time.Hour) +} + +func (e *env) healthFiles(t *testing.T, releases map[string][]string) map[string]NetworkHealth { + t.Helper() + out, err := e.a.NetworkHealthyFiles(context.Background(), releases) + if err != nil { + t.Fatal(err) + } + return out +} + +// Evidence about a file applies to every release the file is part of. +func TestHealthAdviceMatchesFileAcrossReleaseIDs(t *testing.T) { + e := newEnv(t, openRecv) + e.minC = 3 + e.seedFileEv(openNet, relNZB, fileA, 1, nil) + e.seedFileEv(openNet, relNZB, fileA, 2, nil) + e.seedFileEv(openNet, relFiles, fileA, 3, nil) // published by a node that only has the file set + for _, rel := range []string{relNZB, relFiles, hexID(0x13), ""} { + adv, err := e.a.HealthAdvice(context.Background(), rel, fileA, []string{"omicron"}, e.now) + if err != nil { + t.Fatalf("release %q: %v", rel, err) + } + if adv.Kind != AdviceReduceSampleToFloor { + t.Fatalf("release %q: %+v (all three publishers attest the same file)", rel, adv) + } + } + if len(e.store.lastFiles) != 1 || e.store.lastFiles[0] != fileA { + t.Fatalf("lookup was by %v, want the file ID only", e.store.lastFiles) + } +} + +// A publisher that attests one file under many release IDs is still one +// publisher: corroboration is per distinct publisher, never per record. +func TestFileEvidencePublisherCountsOncePerFile(t *testing.T) { + e := newEnv(t, openRecv) + e.minC = 2 + for i := range 6 { + e.seedFileEv(openNet, hexID(byte(0x30+i)), fileA, 1, nil) + } + if adv, _ := e.a.HealthAdvice(context.Background(), relNZB, fileA, []string{"omicron"}, e.now); adv.Kind != AdviceNone { + t.Fatalf("one publisher with six records corroborated itself: %+v", adv) + } + e.seedFileEv(openNet, relFiles, fileA, 2, nil) + if adv, _ := e.a.HealthAdvice(context.Background(), relNZB, fileA, []string{"omicron"}, e.now); adv.Kind != AdviceReduceSampleToFloor { + t.Fatalf("two publishers: %+v", adv) + } +} + +// Every existing rule still applies to evidence found by file: freshness, +// coverage, backbone subset, own records, known publishers, network prefixes. +func TestFileEvidenceKeepsEveryRule(t *testing.T) { + cases := []struct { + name string + mut func(*env) + }{ + {"stale", func(e *env) { + e.seedFileEv(openNet, relFiles, fileA, 1, func(a *database.PeernetAttestation) { a.CheckedAt = e.now.Add(-8 * 24 * time.Hour) }) + }}, + {"future-dated", func(e *env) { + e.seedFileEv(openNet, relFiles, fileA, 1, func(a *database.PeernetAttestation) { a.CheckedAt = e.now.Add(time.Hour) }) + }}, + {"coverage too low", func(e *env) { + e.seedFileEv(openNet, relFiles, fileA, 1, func(a *database.PeernetAttestation) { a.SegmentsChecked = 5 }) + }}, + {"unknown coverage", func(e *env) { + e.seedFileEv(openNet, relFiles, fileA, 1, func(a *database.PeernetAttestation) { a.SegmentsChecked, a.TotalSegments = 0, 0 }) + }}, + {"backbone not a subset", func(e *env) { + e.seedFileEv(openNet, relFiles, fileA, 1, func(a *database.PeernetAttestation) { a.Backbones = []string{"omicron", "zeta"} }) + }}, + {"no backbone", func(e *env) { + e.seedFileEv(openNet, relFiles, fileA, 1, func(a *database.PeernetAttestation) { a.Backbones = nil }) + }}, + {"own record", func(e *env) { + e.seedFileEv(openNet, relFiles, fileA, 1, func(a *database.PeernetAttestation) { a.Source = database.PeernetRecordSourceLocal }) + }}, + {"own node id relayed", func(e *env) { + e.seedFileEv(openNet, relFiles, fileA, 1, func(a *database.PeernetAttestation) { a.PublisherNodeID = self }) + }}, + {"young identity", func(e *env) { + e.seedFileEv(openNet, relFiles, fileA, 1, nil) + e.peers.first[node(1)] = e.now.Add(-time.Hour) + }}, + {"same network prefix", func(e *env) { + e.seedFileEv(openNet, relFiles, fileA, 1, nil) + e.peers.prefix[node(1)] = "10.1.1" + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + e := newEnv(t, openRecv) + e.peers.prefix = map[string]string{} + e.minC = 1 + tc.mut(e) + if tc.name == "same network prefix" { + e.seedFileEv(openNet, relNZB, fileA, 2, nil) + e.peers.prefix[node(2)] = "10.1.1" + e.minC = 2 + } + if adv, _ := e.a.HealthAdvice(context.Background(), relNZB, fileA, []string{"omicron"}, e.now); adv.Kind != AdviceNone { + t.Fatalf("%s: counted as evidence: %+v", tc.name, adv) + } + }) + } +} + +// Another file's evidence is no evidence for this file, whatever the release. +func TestFileEvidenceIsPerFile(t *testing.T) { + e := newEnv(t, openRecv) + e.minC = 1 + e.seedFileEv(openNet, relNZB, fileB, 1, nil) + if adv, _ := e.a.HealthAdvice(context.Background(), relNZB, fileA, []string{"omicron"}, e.now); adv.Kind != AdviceNone { + t.Fatalf("%+v", adv) + } +} + +func TestUnhealthyFileEvidenceUnderAnotherReleaseMovesACheckOnce(t *testing.T) { + e := newEnv(t, openRecv) + e.minC = 2 + e.seedFileEv(openNet, relFiles, fileA, 1, unhealthy) + e.seedFileEv(openNet, relNZB, fileA, 2, unhealthy) + adv, _ := e.a.HealthAdvice(context.Background(), relNZB, fileA, []string{"omicron"}, e.now) + if adv.Kind != AdviceRecheckSooner { + t.Fatalf("%+v", adv) + } + if !e.a.SpendRecheck(relNZB, fileA, adv.EvidenceAt, e.now) { + t.Fatal("budget refused") + } + // The same evidence reached through another release ID is the same + // evidence: it must not be paid for a second time. + again, _ := e.a.HealthAdvice(context.Background(), relFiles, fileA, []string{"omicron"}, e.now) + if again.Kind != AdviceNone { + t.Fatalf("evidence acted on twice, once per release ID: %+v", again) + } + if e.a.SpendRecheck(relFiles, fileA, adv.EvidenceAt, e.now) { + t.Fatal("spent a second token for one piece of evidence") + } +} + +func TestNetworkHealthyFilesNeedsEveryFile(t *testing.T) { + e := newEnv(t, openRecv) + e.minC = 2 + for _, f := range []string{fileA, fileB} { + e.seedFileEv(openNet, relNZB, f, 1, nil) + e.seedFileEv(openNet, relNZB, f, 2, nil) + } + rel := map[string][]string{relNZB: {fileA, fileB}} + if h := e.healthFiles(t, rel)[relNZB]; !h.Healthy || h.HealthyPublishers != 2 { + t.Fatalf("both files corroborated: %+v", h) + } + // A release with a third file nobody attested is not labelled. + if h := e.healthFiles(t, map[string][]string{relNZB: {fileA, fileB, fileC}})[relNZB]; h.Healthy { + t.Fatalf("unattested file: %+v", h) + } + // Evidence for the third file under another release ID counts. + e.seedFileEv(openNet, relFiles, fileC, 1, nil) + e.seedFileEv(openNet, relFiles, fileC, 2, nil) + if h := e.healthFiles(t, map[string][]string{relNZB: {fileA, fileB, fileC}})[relNZB]; !h.Healthy { + t.Fatalf("evidence under another release ID: %+v", h) + } +} + +func TestNetworkHealthyFilesPublisherDoesNotCorroborateViaSecondFile(t *testing.T) { + e := newEnv(t, openRecv) + e.minC = 2 + // Publisher 1 attests both files, publisher 2 only the first: the second + // file has one publisher, which is not corroboration. + e.seedFileEv(openNet, relNZB, fileA, 1, nil) + e.seedFileEv(openNet, relNZB, fileB, 1, nil) + e.seedFileEv(openNet, relNZB, fileA, 2, nil) + if h := e.healthFiles(t, map[string][]string{relNZB: {fileA, fileB}})[relNZB]; h.Healthy { + t.Fatalf("%+v", h) + } +} + +func TestNetworkHealthyFilesUnhealthyFileVetoes(t *testing.T) { + e := newEnv(t, openRecv) + e.minC = 2 + for _, f := range []string{fileA, fileB} { + e.seedFileEv(openNet, relNZB, f, 1, nil) + e.seedFileEv(openNet, relNZB, f, 2, nil) + } + e.seedFileEv(openNet, relFiles, fileB, 3, unhealthy) + if h := e.healthFiles(t, map[string][]string{relNZB: {fileA, fileB}})[relNZB]; !h.Healthy { + t.Fatalf("one unhealthy publisher is below the threshold and must not veto: %+v", h) + } + e.seedFileEv(openNet, relFiles, fileB, 4, unhealthy) + if h := e.healthFiles(t, map[string][]string{relNZB: {fileA, fileB}})[relNZB]; h.Healthy || h.UnhealthyPublishers != 2 { + t.Fatalf("unhealthy evidence at the threshold on one file: %+v", h) + } +} + +func TestNetworkHealthyFilesCoverageAndFreshnessPerFile(t *testing.T) { + e := newEnv(t, openRecv) + e.minC = 1 + e.seedFileEv(openNet, relNZB, fileA, 1, nil) + e.seedFileEv(openNet, relNZB, fileB, 1, func(a *database.PeernetAttestation) { a.SegmentsChecked = 3 }) + if h := e.healthFiles(t, map[string][]string{relNZB: {fileA, fileB}})[relNZB]; h.Healthy { + t.Fatalf("insufficient coverage on one file: %+v", h) + } + e2 := newEnv(t, openRecv) + e2.minC = 1 + e2.seedFileEv(openNet, relNZB, fileA, 1, func(a *database.PeernetAttestation) { a.CheckedAt = e2.now.Add(-8 * 24 * time.Hour) }) + if h := e2.healthFiles(t, map[string][]string{relNZB: {fileA}})[relNZB]; h.Healthy { + t.Fatalf("stale: %+v", h) + } +} + +// The release label never costs more than one attestation lookup per network +// for the whole batch, and bounds both the files per release and in total. +func TestNetworkHealthyFilesIsOneBoundedBatch(t *testing.T) { + e := newEnv(t, openRecv, privRecv) + e.minC = 1 + releases := map[string][]string{} + for i := range 100 { + rel, f := hexID(byte(0x40+i%100)), hexID(byte(0x80+i%100)) + releases[rel] = []string{f} + e.seedFileEv(openNet, rel, f, 1, nil) + } + out := e.healthFiles(t, releases) + if len(out) != 100 { + t.Fatalf("every requested release is present: %d", len(out)) + } + if e.store.calls["att"] != 2 { + t.Fatalf("%d attestation lookups for two networks, want 2", e.store.calls["att"]) + } + healthy := 0 + for _, h := range out { + if h.Healthy { + healthy++ + } + } + if healthy != 100 { + t.Fatalf("%d healthy", healthy) + } + + var big []string + for i := range MaxLabelFilesPerRelease + 1 { + big = append(big, hexID(byte(0xa0+i))) + } + for _, f := range big { + e.seedFileEv(openNet, hexID(0x90), f, 1, nil) + } + if h := e.healthFiles(t, map[string][]string{hexID(0x90): big})[hexID(0x90)]; h.Healthy { + t.Fatal("a release with more files than the cap must not be labelled") + } + if h := e.healthFiles(t, map[string][]string{hexID(0x91): nil})[hexID(0x91)]; h.Healthy { + t.Fatal("a release with no known files must not be labelled") + } + if len(e.store.lastFiles) > MaxLabelFiles { + t.Fatalf("looked up %d files in one query", len(e.store.lastFiles)) + } +} + +func TestNetworkHealthyFilesValidatesInput(t *testing.T) { + e := newEnv(t, openRecv) + if _, err := e.a.NetworkHealthyFiles(context.Background(), map[string][]string{"nope": {fileA}}); err == nil { + t.Fatal("bad release id accepted") + } + if _, err := e.a.NetworkHealthyFiles(context.Background(), map[string][]string{relNZB: {"nothex"}}); err == nil { + t.Fatal("bad file id accepted") + } + many := map[string][]string{} + for i := 0; i <= maxReleaseIDs; i++ { + many[hexID(byte(i))] = []string{fileA} + } + if _, err := e.a.NetworkHealthyFiles(context.Background(), many); err == nil { + t.Fatal("more releases than the bound accepted") + } +} + +// A publisher's own attestation is not independent evidence for its own offer. +func TestNetworkHealthyFilesExcludingLeavesOutThePublishersOwnEvidence(t *testing.T) { + e := newEnv(t, openRecv) + e.minC = 2 + e.seedFileEv(openNet, relNZB, fileA, 1, nil) + e.seedFileEv(openNet, relNZB, fileA, 2, nil) + rel := map[string][]string{relNZB: {fileA}} + if h := e.healthFiles(t, rel)[relNZB]; !h.Healthy || h.HealthyPublishers != 2 { + t.Fatalf("baseline: %+v", h) + } + got, err := e.a.NetworkHealthyFilesExcluding(context.Background(), node(1), rel) + if err != nil { + t.Fatal(err) + } + if h := got[relNZB]; h.Healthy || h.HealthyPublishers != 1 { + t.Fatalf("own attestation still counted: %+v", h) + } + got, err = e.a.NetworkHealthyFilesExcluding(context.Background(), "", rel) + if err != nil || !got[relNZB].Healthy { + t.Fatalf("empty exclude must exclude nobody: %+v %v", got[relNZB], err) + } +} + +func TestNetworkHealthyExcludingLeavesOutThePublishersOwnEvidence(t *testing.T) { + e := newEnv(t, openRecv) + e.minC = 2 + e.seed(openNet, 1, nil) + e.seed(openNet, 2, nil) + got, err := e.a.NetworkHealthyExcluding(context.Background(), node(1), release) + if err != nil { + t.Fatal(err) + } + if h := got[release]; h.Healthy || h.HealthyPublishers != 1 { + t.Fatalf("own attestation still counted: %+v", h) + } + got, err = e.a.NetworkHealthy(context.Background(), release) + if err != nil || !got[release].Healthy || got[release].HealthyPublishers != 2 { + t.Fatalf("without exclusion both count: %+v %v", got[release], err) + } +} diff --git a/internal/peernet/hints/hints.go b/internal/peernet/hints/hints.go index a988d225b..a77485af4 100644 --- a/internal/peernet/hints/hints.go +++ b/internal/peernet/hints/hints.go @@ -37,8 +37,6 @@ package hints import ( "context" "errors" - "fmt" - "sort" "strings" "sync" "time" @@ -60,9 +58,14 @@ const ( maxCandidates = database.PeernetCandidatesMaxLimit maxReleaseIDs = database.PeernetMaxReleaseIDsPerQuery maxNetworks = database.PeernetMaxNetworksPerQuery - maxPublisherLookups = 64 - maxAdvised = 4096 - prefetchChunk = 200 + // MaxLabelFilesPerRelease is how many main (video) files of one release the + // network-health label looks at; a release with more is not labelled. + MaxLabelFilesPerRelease = 16 + // MaxLabelFiles bounds the distinct files of one labelling call. + MaxLabelFiles = database.PeernetMaxFilesPerQuery + maxPublisherLookups = 64 + maxAdvised = 4096 + prefetchChunk = 200 // MinCoverageSegments and MinCoveragePercent set how much of a file a // healthy attestation must have checked to count as evidence: at least @@ -138,6 +141,10 @@ type Network struct { // *database.PeernetRecordRepository. type Store interface { AttestationsFor(ctx context.Context, networkID, releaseID, fileID string, notBefore time.Time) ([]database.PeernetAttestation, error) + // AttestationsForFiles returns attestations of the given files whatever + // release they were published under (at most one row per file, publisher + // and verdict). + AttestationsForFiles(ctx context.Context, networkID string, fileIDs []string, notBefore time.Time) ([]database.PeernetAttestation, error) HealthSummaryForReleases(ctx context.Context, networkID string, releaseIDs []string, notBefore time.Time) (map[string]database.PeernetHealthSummary, error) CandidatesByMedia(ctx context.Context, networkIDs []string, imdbID string, season, episode *int, limit int) ([]database.PeernetCandidate, error) } @@ -303,558 +310,3 @@ func relation(att []string, local map[string]struct{}) (subset, overlap bool) { } func validID(s string) bool { return netid.IsHexID(s, true) } - -type adviceEvidence struct { - openHealthy int - privHealthy int - privNewest time.Time - privNet netid.NetworkID - openNet netid.NetworkID - unhealthyNewest time.Time -} - -// HealthAdvice returns what peer evidence allows for one file. localBackbones -// are the raw backbone labels of the local providers. now is the caller's -// current time (normally the injected clock). Errors from the store or the -// peer table are returned with AdviceNone: advice is never guessed. -func (a *Advisor) HealthAdvice(ctx context.Context, releaseID, fileID string, localBackbones []string, now time.Time) (Advice, error) { - if !validID(releaseID) || !validID(fileID) { - return Advice{}, errors.New("hints: release and file ids must be 64 lower-case hex characters") - } - local := normaliseSet(localBackbones) - if len(local) == 0 { - return Advice{}, nil - } - var ev adviceEvidence - for _, n := range a.receiving() { - if err := a.gather(ctx, n, releaseID, fileID, local, now, &ev); err != nil { - return Advice{}, err - } - } - return a.decide(releaseID+"|"+fileID, ev, now), nil -} - -func (a *Advisor) gather(ctx context.Context, n Network, releaseID, fileID string, local map[string]struct{}, now time.Time, ev *adviceEvidence) error { - atts, err := a.cfg.Store.AttestationsFor(ctx, n.ID.String(), releaseID, fileID, now.Add(-records.AttestationValidity)) - if err != nil { - return fmt.Errorf("hints: attestations: %w", err) - } - healthy := map[string]struct{}{} // distinct groups attesting healthy - unhealthyG := map[string]struct{}{} // distinct groups attesting unhealthy - seenHealthy := map[string]struct{}{} - known := newKnownCache(a, ctx, n, now) - if err := known.prefetch(a.relevantPublishers(atts, fileID, local, now)); err != nil { - return err - } - var newest, unhealthyNewest time.Time - for _, att := range atts { - if !strings.EqualFold(att.FileID, fileID) || !a.usable(att, now) { - continue - } - subset, overlap := relation(att.Backbones, local) - if !att.Healthy { - if !overlap { - continue - } - // The same standing as healthy evidence: on the open network a - // publisher known for MinKnown, so fresh identities cannot drain - // the re-check budget. - ok, err := known.eligible(att.PublisherNodeID) - if err != nil { - return err - } - if ok { - unhealthyG[known.groupOf(att.PublisherNodeID)] = struct{}{} - if att.CheckedAt.After(unhealthyNewest) { - unhealthyNewest = att.CheckedAt - } - } - continue - } - if !subset || !CoverageSufficient(att.SegmentsChecked, att.TotalSegments) { - continue - } - if _, dup := seenHealthy[att.PublisherNodeID]; dup { - if att.CheckedAt.After(newest) { - newest = att.CheckedAt - } - continue - } - ok, err := known.eligible(att.PublisherNodeID) - if err != nil { - return err - } - if ok { - seenHealthy[att.PublisherNodeID] = struct{}{} - healthy[known.groupOf(att.PublisherNodeID)] = struct{}{} - if att.CheckedAt.After(newest) { - newest = att.CheckedAt - } - } - } - // Unhealthy evidence needs the same number of distinct publishers as - // healthy corroboration, so a lone identity cannot veto or force work. - if len(unhealthyG) >= a.threshold(n) && unhealthyNewest.After(ev.unhealthyNewest) { - ev.unhealthyNewest = unhealthyNewest - } - if n.ID.IsOpen() { - ev.openHealthy += len(healthy) - ev.openNet = n.ID - } else if len(healthy) >= a.cfg.PrivateThreshold && newest.After(ev.privNewest) { - ev.privHealthy += len(healthy) - ev.privNewest, ev.privNet = newest, n.ID - } - return nil -} - -// relevantPublishers lists, in order and without repeats, the publishers -// whose attestations gather may count, so their standing can be fetched in -// one go. -func (a *Advisor) relevantPublishers(atts []database.PeernetAttestation, fileID string, local map[string]struct{}, now time.Time) []string { - var out []string - seen := map[string]struct{}{} - for _, att := range atts { - if !strings.EqualFold(att.FileID, fileID) || !a.usable(att, now) { - continue - } - subset, overlap := relation(att.Backbones, local) - if att.Healthy && (!subset || !CoverageSufficient(att.SegmentsChecked, att.TotalSegments)) || !att.Healthy && !overlap { - continue - } - if _, dup := seen[att.PublisherNodeID]; !dup { - seen[att.PublisherNodeID] = struct{}{} - out = append(out, att.PublisherNodeID) - } - } - return out -} - -func (a *Advisor) decide(fileKey string, ev adviceEvidence, now time.Time) Advice { - if !ev.unhealthyNewest.IsZero() { - // Unhealthy evidence, even next to healthy evidence, never shortens - // anything; at most it may move a check forward, once. Deciding is - // read-only: the budget is spent by SpendRecheck when a check really - // was moved. - if a.acted(fileKey, ev.unhealthyNewest) { - return Advice{} - } - return Advice{Kind: AdviceRecheckSooner, EvidenceAt: ev.unhealthyNewest} - } - if ev.privHealthy >= a.cfg.PrivateThreshold && !ev.privNewest.IsZero() { - if until := ev.privNewest.Add(records.AttestationValidity); until.After(now) { - return Advice{Kind: AdviceMaySkipOneCheck, SkipUntil: until, Network: ev.privNet} - } - } - if ev.openHealthy >= a.corroboration() { - return Advice{Kind: AdviceReduceSampleToFloor, Network: ev.openNet} - } - return Advice{} -} - -// acted reports whether evidence of the given check time (or newer) for the -// file was already acted on. -func (a *Advisor) acted(key string, newest time.Time) bool { - a.mu.Lock() - defer a.mu.Unlock() - prev, seen := a.advised[key] - return seen && !newest.After(prev) -} - -// markAdvised records acted-on evidence, evicting the entry with the oldest -// evidence when the memory is full. The caller holds a.mu. -func (a *Advisor) markAdvised(key string, newest time.Time) { - if _, have := a.advised[key]; !have && len(a.advised) >= maxAdvised { - var oldestKey string - var oldest time.Time - first := true - for k, t := range a.advised { - if first || t.Before(oldest) { - oldestKey, oldest, first = k, t, false - } - } - delete(a.advised, oldestKey) - } - a.advised[key] = newest -} - -// forget clears the memory of acted-on evidence. -func (a *Advisor) forget() { - a.mu.Lock() - a.advised = map[string]time.Time{} - a.mu.Unlock() -} - -// knownCache answers per-call "may this publisher count" questions: always -// yes in a private network, and in the open network only for nodes known for -// MinKnown, with a bounded number of lookups per call. On the open network it -// also tells which network prefix a publisher was verified at, so identities -// sharing a prefix can be counted once. -type knownCache struct { - a *Advisor - ctx context.Context - n Network - now time.Time - res map[string]bool - prefix map[string]string - lookups int -} - -func newKnownCache(a *Advisor, ctx context.Context, n Network, now time.Time) *knownCache { - return &knownCache{a: a, ctx: ctx, n: n, now: now, res: map[string]bool{}, prefix: map[string]string{}} -} - -// groupOf is the key distinct publishers are counted by: the publisher itself -// in a private network, its verified network prefix on the open one. -func (k *knownCache) groupOf(node string) string { - if p, ok := k.prefix[node]; ok { - return p - } - return node -} - -// prefetch resolves the standing of many publishers with one batch lookup -// per chunk when the peer lookup supports it. Results are exact, so the -// per-call lookup bound (which only guards one-by-one lookups) cannot let -// fresh identities push known publishers out. -func (k *knownCache) prefetch(nodes []string) error { - if !k.n.ID.IsOpen() || len(nodes) == 0 { - return nil - } - pre, hasPrefixes := k.a.cfg.Peers.(PeerPrefixes) - batch, hasBatch := k.a.cfg.Peers.(PeerSeenBatch) - if !hasPrefixes && !hasBatch { - return nil - } - for start := 0; start < len(nodes); start += prefetchChunk { - chunk := nodes[start:min(start+prefetchChunk, len(nodes))] - if hasPrefixes { - got, err := pre.KnownPrefixes(k.ctx, k.n.ID.String(), chunk, k.a.cfg.MinKnown, k.now) - if err != nil { - return fmt.Errorf("hints: peer first-seen: %w", err) - } - for _, n := range chunk { - p, ok := got[n] - k.res[n] = ok && p != "" - if k.res[n] { - k.prefix[n] = p - } - } - continue - } - got, err := batch.KnownAmong(k.ctx, k.n.ID.String(), chunk, k.a.cfg.MinKnown, k.now) - if err != nil { - return fmt.Errorf("hints: peer first-seen: %w", err) - } - for _, n := range chunk { - k.res[n] = got[n] - } - } - return nil -} - -func (k *knownCache) eligible(node string) (bool, error) { - if !k.n.ID.IsOpen() { - return true, nil - } - if v, ok := k.res[node]; ok { - return v, nil - } - if k.lookups >= maxPublisherLookups { - return false, nil - } - k.lookups++ - ok, err := k.a.cfg.Peers.KnownFor(k.ctx, k.n.ID.String(), node, k.a.cfg.MinKnown, k.now) - if err != nil { - return false, fmt.Errorf("hints: peer first-seen: %w", err) - } - k.res[node] = ok - return ok, nil -} - -// NetworkHealth is the network-wide health label of one release. It is never -// a status. -type NetworkHealth struct { - // Healthy: on some receiving network enough eligible publishers attest it - // healthy and unhealthy evidence does not itself reach the threshold on - // any network. - Healthy bool - // HealthyPublishers and UnhealthyPublishers count eligible publishers - // (distinct network prefixes on the open network), summed over networks - // (informational). - HealthyPublishers int - UnhealthyPublishers int - // Newest is the latest check time among eligible healthy attestations. - Newest time.Time -} - -// NetworkHealthy labels releases for candidate lists (trust rule 5). Every -// requested release is present in the result. At most 200 distinct IDs are -// accepted per call. -func (a *Advisor) NetworkHealthy(ctx context.Context, releaseIDs ...string) (map[string]NetworkHealth, error) { - ids := make([]string, 0, len(releaseIDs)) - seen := make(map[string]struct{}, len(releaseIDs)) - for _, id := range releaseIDs { - if !validID(id) { - return nil, errors.New("hints: release ids must be 64 lower-case hex characters") - } - if _, dup := seen[id]; !dup { - seen[id] = struct{}{} - ids = append(ids, id) - } - } - if len(ids) > maxReleaseIDs { - return nil, ErrTooManyIDs - } - out := make(map[string]NetworkHealth, len(ids)) - anyHealthy := make(map[string]bool, len(ids)) - vetoed := make(map[string]bool, len(ids)) - for _, id := range ids { - out[id] = NetworkHealth{} - } - if len(ids) == 0 { - return out, nil - } - now := a.cfg.Now() - for _, n := range a.receiving() { - if err := a.labelNetwork(ctx, n, ids, now, out, anyHealthy, vetoed); err != nil { - return nil, err - } - } - for id, h := range out { - h.Healthy = anyHealthy[id] && !vetoed[id] - out[id] = h - } - return out, nil -} - -func (a *Advisor) labelNetwork(ctx context.Context, n Network, ids []string, now time.Time, out map[string]NetworkHealth, anyHealthy, vetoed map[string]bool) error { - notBefore := now.Add(-records.AttestationValidity) - summary, err := a.cfg.Store.HealthSummaryForReleases(ctx, n.ID.String(), ids, notBefore) - if err != nil { - return fmt.Errorf("hints: health summary: %w", err) - } - for _, id := range ids { - if _, ok := summary[id]; !ok { - continue - } - atts, err := a.cfg.Store.AttestationsFor(ctx, n.ID.String(), id, "", notBefore) - if err != nil { - return fmt.Errorf("hints: attestations: %w", err) - } - hE, uE, newest, err := a.tally(ctx, n, atts, now) - if err != nil { - return err - } - h := out[id] - h.HealthyPublishers += hE - h.UnhealthyPublishers += uE - if newest.After(h.Newest) { - h.Newest = newest - } - out[id] = h - // A lone unhealthy publisher does not veto: unhealthy evidence has to - // meet the same threshold as healthy corroboration. - if uE >= a.threshold(n) { - vetoed[id] = true - } - if hE >= a.threshold(n) { - anyHealthy[id] = true - } - } - return nil -} - -// tally counts eligible publishers, one per distinct group (see groupOf). A -// publisher that attested anything unhealthy counts as unhealthy only. -func (a *Advisor) tally(ctx context.Context, n Network, atts []database.PeernetAttestation, now time.Time) (healthy, unhealthy int, newest time.Time, err error) { - type verdict struct { - bad bool - newest time.Time - } - order := []string{} - per := map[string]*verdict{} - for _, att := range atts { - if !a.usable(att, now) { - continue - } - v := per[att.PublisherNodeID] - if v == nil { - v = &verdict{} - per[att.PublisherNodeID] = v - order = append(order, att.PublisherNodeID) - } - if !att.Healthy { - v.bad = true - } else if CoverageSufficient(att.SegmentsChecked, att.TotalSegments) && att.CheckedAt.After(v.newest) { - v.newest = att.CheckedAt - } - } - known := newKnownCache(a, ctx, n, now) - if err := known.prefetch(order); err != nil { - return 0, 0, time.Time{}, err - } - healthyG, unhealthyG := map[string]struct{}{}, map[string]struct{}{} - for _, node := range order { - ok, kerr := known.eligible(node) - if kerr != nil { - return 0, 0, time.Time{}, kerr - } - if !ok { - continue - } - v := per[node] - g := known.groupOf(node) - switch { - case v.bad: - unhealthyG[g] = struct{}{} - case !v.newest.IsZero(): - healthyG[g] = struct{}{} - if v.newest.After(newest) { - newest = v.newest - } - } - } - return len(healthyG), len(unhealthyG), newest, nil -} - -// Candidate is one release found for a media key. Title is peer-supplied data -// to be rendered as text only. -type Candidate struct { - ReleaseID string - Title string - TotalSize int64 - CreatedAt time.Time - // Networks lists every receiving network the release was seen in, sorted. - Networks []netid.NetworkID - // Own marks releases this node published itself. - Own bool - Health NetworkHealth -} - -func validIMDb(s string) bool { - if len(s) < 3 || len(s) > 2+maxIMDbDigits || s[:2] != "tt" { - return false - } - for i := 2; i < len(s); i++ { - if s[i] < '0' || s[i] > '9' { - return false - } - } - return true -} - -// Candidates returns the release records known for a media key across the -// networks this node receives from, one entry per release ID, healthy-labelled -// first, then newest, then by release ID; at most CandidateLimit entries. A -// nil season and episode match movies only (see the store contract). -func (a *Advisor) Candidates(ctx context.Context, imdbID string, season, episode *int) ([]Candidate, error) { - if !validIMDb(imdbID) { - return nil, errors.New("hints: invalid imdb id") - } - if (season != nil && *season < 0) || (episode != nil && *episode < 0) { - return nil, errors.New("hints: negative season or episode") - } - nets := a.receiving() - if len(nets) == 0 { - return nil, nil - } - idStrs := make([]string, len(nets)) - for i, n := range nets { - idStrs[i] = n.ID.String() - } - rows, err := a.cfg.Store.CandidatesByMedia(ctx, idStrs, imdbID, season, episode, a.cfg.CandidateLimit) - if err != nil { - return nil, fmt.Errorf("hints: candidates: %w", err) - } - merged := a.merge(rows) - if len(merged) == 0 { - return nil, nil - } - ids := make([]string, len(merged)) - for i, c := range merged { - ids[i] = c.ReleaseID - } - health, err := a.NetworkHealthy(ctx, ids...) - if err != nil { - return nil, err - } - for i := range merged { - merged[i].Health = health[merged[i].ReleaseID] - } - sort.Slice(merged, func(i, j int) bool { - x, y := merged[i], merged[j] - switch { - case x.Health.Healthy != y.Health.Healthy: - return x.Health.Healthy - case !x.CreatedAt.Equal(y.CreatedAt): - return x.CreatedAt.After(y.CreatedAt) - } - return x.ReleaseID < y.ReleaseID - }) - if len(merged) > a.cfg.CandidateLimit { - merged = merged[:a.cfg.CandidateLimit] - } - return merged, nil -} - -// merge de-duplicates rows by release ID; the newest record defines the entry. -func (a *Advisor) merge(rows []database.PeernetCandidate) []Candidate { - byID := map[string]*Candidate{} - var order []string - for _, r := range rows { - nw, err := netid.ParseNetworkID(r.NetworkID) - title := records.CleanTitle(r.Title) - if err != nil || !validID(r.ReleaseID) || title == "" { - continue - } - own := r.Source == database.PeernetRecordSourceLocal || strings.EqualFold(r.PublisherNodeID, a.cfg.SelfNodeID) - c := byID[r.ReleaseID] - if c == nil { - c = &Candidate{ReleaseID: r.ReleaseID} - byID[r.ReleaseID] = c - order = append(order, r.ReleaseID) - } - if c.Title == "" || r.CreatedAt.After(c.CreatedAt) { - c.Title, c.TotalSize, c.CreatedAt = title, r.TotalSize, r.CreatedAt - } - c.Own = c.Own || own - if !containsNet(c.Networks, nw) { - c.Networks = append(c.Networks, nw) - } - } - out := make([]Candidate, 0, len(order)) - for _, id := range order { - c := byID[id] - sort.Slice(c.Networks, func(i, j int) bool { return c.Networks[i].String() < c.Networks[j].String() }) - out = append(out, *c) - } - return out -} - -func containsNet(list []netid.NetworkID, n netid.NetworkID) bool { - for _, x := range list { - if x == n { - return true - } - } - return false -} - -// SpendRecheck pays one token of the shared budget for moving a check forward -// on the unhealthy evidence of the given check time, and remembers that the -// evidence was acted on. It reports false when this evidence was already acted -// on or no token is left. It is called only once a check really was moved: -// HealthAdvice itself spends nothing. -func (a *Advisor) SpendRecheck(releaseID, fileID string, evidenceAt, now time.Time) bool { - key := releaseID + "|" + fileID - a.mu.Lock() - defer a.mu.Unlock() - if prev, seen := a.advised[key]; seen && !evidenceAt.After(prev) { - return false - } - if !a.cfg.Budget.Take(now) { - return false - } - a.markAdvised(key, evidenceAt) - return true -} diff --git a/internal/peernet/hints/hints_test.go b/internal/peernet/hints/hints_test.go index b39ce98d0..d13365577 100644 --- a/internal/peernet/hints/hints_test.go +++ b/internal/peernet/hints/hints_test.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "errors" "fmt" + "sort" "sync" "testing" "time" @@ -51,6 +52,7 @@ type fakeStore struct { lastLimit int lastNets []string lastSeason *int + lastFiles []string } func newStore() *fakeStore { @@ -73,6 +75,35 @@ func (s *fakeStore) AttestationsFor(_ context.Context, nw, rel, _ string, _ time return append([]database.PeernetAttestation(nil), s.atts[nw+"|"+rel]...), nil } +// AttestationsForFiles matches by file across every release key of the +// network, like the real store; freshness is left to the advisor. +func (s *fakeStore) AttestationsForFiles(_ context.Context, nw string, files []string, _ time.Time) ([]database.PeernetAttestation, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.calls["att"]++ + s.lastFiles = append([]string(nil), files...) + if s.err != nil { + return nil, s.err + } + want := map[string]bool{} + for _, f := range files { + want[f] = true + } + var out []database.PeernetAttestation + for k, list := range s.atts { + if len(k) < len(nw)+1 || k[:len(nw)+1] != nw+"|" { + continue + } + for _, a := range list { + if want[a.FileID] { + out = append(out, a) + } + } + } + sort.SliceStable(out, func(i, j int) bool { return out[i].PublisherNodeID < out[j].PublisherNodeID }) + return out, nil +} + func (s *fakeStore) HealthSummaryForReleases(_ context.Context, nw string, ids []string, notBefore time.Time) (map[string]database.PeernetHealthSummary, error) { s.mu.Lock() defer s.mu.Unlock() @@ -523,6 +554,7 @@ func TestManyIdentitiesCannotRaiseTheRecheckBudget(t *testing.T) { granted := 0 for f := 0; f < 40; f++ { rel := hexID(byte(100 + f)) + file := hexID(byte(100 + f)) // evidence is per file: forty distinct files for i := 0; i < 50; i++ { nw := openNet if i%2 == 0 { @@ -593,6 +625,7 @@ func TestBudgetIsSharedAcrossFiles(t *testing.T) { e.minC = 1 for i := 0; i < 5; i++ { rel := hexID(byte(10 + i)) + file := hexID(byte(10 + i)) // evidence is per file: five distinct files e.peers.first[node(i)] = e.now.Add(-7 * 24 * time.Hour) e.store.add(openNet, rel, database.PeernetAttestation{ PublisherNodeID: node(i), FileID: file, Healthy: false, Backbones: []string{"omicron"}, @@ -601,6 +634,7 @@ func TestBudgetIsSharedAcrossFiles(t *testing.T) { } n := 0 for i := 0; i < 5; i++ { + file := hexID(byte(10 + i)) adv, _ := e.a.HealthAdvice(context.Background(), hexID(byte(10+i)), file, []string{"omicron"}, e.now) if adv.Kind == AdviceRecheckSooner && e.a.SpendRecheck(hexID(byte(10+i)), file, adv.EvidenceAt, e.now) { n++ @@ -610,8 +644,8 @@ func TestBudgetIsSharedAcrossFiles(t *testing.T) { t.Fatalf("%d", n) } e.now = e.now.Add(21 * time.Minute) // 3/hour refills one token every 20 minutes - adv, _ := e.a.HealthAdvice(context.Background(), hexID(14), file, []string{"omicron"}, e.now) - if adv.Kind != AdviceRecheckSooner || !e.a.SpendRecheck(hexID(14), file, adv.EvidenceAt, e.now) { + adv, _ := e.a.HealthAdvice(context.Background(), hexID(14), hexID(14), []string{"omicron"}, e.now) + if adv.Kind != AdviceRecheckSooner || !e.a.SpendRecheck(hexID(14), hexID(14), adv.EvidenceAt, e.now) { t.Fatalf("budget did not refill: %+v", adv) } } diff --git a/internal/peernet/hints/knowncache.go b/internal/peernet/hints/knowncache.go new file mode 100644 index 000000000..5fe35bd38 --- /dev/null +++ b/internal/peernet/hints/knowncache.go @@ -0,0 +1,94 @@ +package hints + +import ( + "context" + "fmt" + "time" +) + +// knownCache answers per-call "may this publisher count" questions: always +// yes in a private network, and in the open network only for nodes known for +// MinKnown, with a bounded number of lookups per call. On the open network it +// also tells which network prefix a publisher was verified at, so identities +// sharing a prefix can be counted once. +type knownCache struct { + a *Advisor + ctx context.Context + n Network + now time.Time + res map[string]bool + prefix map[string]string + lookups int +} + +func newKnownCache(a *Advisor, ctx context.Context, n Network, now time.Time) *knownCache { + return &knownCache{a: a, ctx: ctx, n: n, now: now, res: map[string]bool{}, prefix: map[string]string{}} +} + +// groupOf is the key distinct publishers are counted by: the publisher itself +// in a private network, its verified network prefix on the open one. +func (k *knownCache) groupOf(node string) string { + if p, ok := k.prefix[node]; ok { + return p + } + return node +} + +// prefetch resolves the standing of many publishers with one batch lookup +// per chunk when the peer lookup supports it. Results are exact, so the +// per-call lookup bound (which only guards one-by-one lookups) cannot let +// fresh identities push known publishers out. +func (k *knownCache) prefetch(nodes []string) error { + if !k.n.ID.IsOpen() || len(nodes) == 0 { + return nil + } + pre, hasPrefixes := k.a.cfg.Peers.(PeerPrefixes) + batch, hasBatch := k.a.cfg.Peers.(PeerSeenBatch) + if !hasPrefixes && !hasBatch { + return nil + } + for start := 0; start < len(nodes); start += prefetchChunk { + chunk := nodes[start:min(start+prefetchChunk, len(nodes))] + if hasPrefixes { + got, err := pre.KnownPrefixes(k.ctx, k.n.ID.String(), chunk, k.a.cfg.MinKnown, k.now) + if err != nil { + return fmt.Errorf("hints: peer first-seen: %w", err) + } + for _, n := range chunk { + p, ok := got[n] + k.res[n] = ok && p != "" + if k.res[n] { + k.prefix[n] = p + } + } + continue + } + got, err := batch.KnownAmong(k.ctx, k.n.ID.String(), chunk, k.a.cfg.MinKnown, k.now) + if err != nil { + return fmt.Errorf("hints: peer first-seen: %w", err) + } + for _, n := range chunk { + k.res[n] = got[n] + } + } + return nil +} + +func (k *knownCache) eligible(node string) (bool, error) { + if !k.n.ID.IsOpen() { + return true, nil + } + if v, ok := k.res[node]; ok { + return v, nil + } + if k.lookups >= maxPublisherLookups { + return false, nil + } + k.lookups++ + ok, err := k.a.cfg.Peers.KnownFor(k.ctx, k.n.ID.String(), node, k.a.cfg.MinKnown, k.now) + if err != nil { + return false, fmt.Errorf("hints: peer first-seen: %w", err) + } + k.res[node] = ok + return ok, nil +} diff --git a/internal/peernet/hints/label.go b/internal/peernet/hints/label.go new file mode 100644 index 000000000..793766bf2 --- /dev/null +++ b/internal/peernet/hints/label.go @@ -0,0 +1,370 @@ +package hints + +import ( + "context" + "errors" + "fmt" + "sort" + "time" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/peernet/records" +) + +// NetworkHealth is the network-wide health label of one release. It is never +// a status. +type NetworkHealth struct { + // Healthy: on some receiving network enough eligible publishers attest it + // healthy and unhealthy evidence does not itself reach the threshold on + // any network. + Healthy bool + // HealthyPublishers and UnhealthyPublishers count eligible publishers + // (distinct network prefixes on the open network), summed over networks + // (informational). + HealthyPublishers int + UnhealthyPublishers int + // Newest is the latest check time among eligible healthy attestations. + Newest time.Time +} + +// NetworkHealthy labels releases for candidate lists (trust rule 5). Every +// requested release is present in the result. At most 200 distinct IDs are +// accepted per call. +func (a *Advisor) NetworkHealthy(ctx context.Context, releaseIDs ...string) (map[string]NetworkHealth, error) { + return a.NetworkHealthyExcluding(ctx, "", releaseIDs...) +} + +// NetworkHealthyExcluding is NetworkHealthy with every attestation of the +// publisher exclude (a node ID, 64 lower-case hex characters; empty excludes +// nobody) left out of the evidence. A publisher's own attestation is not +// independent evidence for its own offer. +func (a *Advisor) NetworkHealthyExcluding(ctx context.Context, exclude string, releaseIDs ...string) (map[string]NetworkHealth, error) { + ids := make([]string, 0, len(releaseIDs)) + seen := make(map[string]struct{}, len(releaseIDs)) + for _, id := range releaseIDs { + if !validID(id) { + return nil, errors.New("hints: release ids must be 64 lower-case hex characters") + } + if _, dup := seen[id]; !dup { + seen[id] = struct{}{} + ids = append(ids, id) + } + } + if len(ids) > maxReleaseIDs { + return nil, ErrTooManyIDs + } + out := make(map[string]NetworkHealth, len(ids)) + anyHealthy := make(map[string]bool, len(ids)) + vetoed := make(map[string]bool, len(ids)) + for _, id := range ids { + out[id] = NetworkHealth{} + } + if len(ids) == 0 { + return out, nil + } + now := a.cfg.Now() + for _, n := range a.receiving() { + if err := a.labelNetwork(ctx, n, ids, now, exclude, out, anyHealthy, vetoed); err != nil { + return nil, err + } + } + for id, h := range out { + h.Healthy = anyHealthy[id] && !vetoed[id] + out[id] = h + } + return out, nil +} + +func (a *Advisor) labelNetwork(ctx context.Context, n Network, ids []string, now time.Time, exclude string, out map[string]NetworkHealth, anyHealthy, vetoed map[string]bool) error { + notBefore := now.Add(-records.AttestationValidity) + summary, err := a.cfg.Store.HealthSummaryForReleases(ctx, n.ID.String(), ids, notBefore) + if err != nil { + return fmt.Errorf("hints: health summary: %w", err) + } + for _, id := range ids { + if _, ok := summary[id]; !ok { + continue + } + atts, err := a.cfg.Store.AttestationsFor(ctx, n.ID.String(), id, "", notBefore) + if err != nil { + return fmt.Errorf("hints: attestations: %w", err) + } + hE, uE, newest, err := a.tally(ctx, n, withoutPublisher(atts, exclude), now) + if err != nil { + return err + } + h := out[id] + h.HealthyPublishers += hE + h.UnhealthyPublishers += uE + if newest.After(h.Newest) { + h.Newest = newest + } + out[id] = h + // A lone unhealthy publisher does not veto: unhealthy evidence has to + // meet the same threshold as healthy corroboration. + if uE >= a.threshold(n) { + vetoed[id] = true + } + if hE >= a.threshold(n) { + anyHealthy[id] = true + } + } + return nil +} + +// tally counts eligible publishers, one per distinct group (see groupOf). A +// publisher that attested anything unhealthy counts as unhealthy only. +func (a *Advisor) tally(ctx context.Context, n Network, atts []database.PeernetAttestation, now time.Time) (healthy, unhealthy int, newest time.Time, err error) { + known := newKnownCache(a, ctx, n, now) + if err := known.prefetch(publishersOf(atts, a, now)); err != nil { + return 0, 0, time.Time{}, err + } + return a.tallyWith(known, atts, now) +} + +// publishersOf lists the usable publishers of atts in order, without repeats. +func publishersOf(atts []database.PeernetAttestation, a *Advisor, now time.Time) []string { + var order []string + seen := map[string]struct{}{} + for _, att := range atts { + if !a.usable(att, now) { + continue + } + if _, dup := seen[att.PublisherNodeID]; !dup { + seen[att.PublisherNodeID] = struct{}{} + order = append(order, att.PublisherNodeID) + } + } + return order +} + +// tallyWith is tally over a knownCache the caller already filled, so many +// files can share one batch of standing lookups. +func (a *Advisor) tallyWith(known *knownCache, atts []database.PeernetAttestation, now time.Time) (healthy, unhealthy int, newest time.Time, err error) { + type verdict struct { + bad bool + newest time.Time + } + order := []string{} + per := map[string]*verdict{} + for _, att := range atts { + if !a.usable(att, now) { + continue + } + v := per[att.PublisherNodeID] + if v == nil { + v = &verdict{} + per[att.PublisherNodeID] = v + order = append(order, att.PublisherNodeID) + } + if !att.Healthy { + v.bad = true + } else if CoverageSufficient(att.SegmentsChecked, att.TotalSegments) && att.CheckedAt.After(v.newest) { + v.newest = att.CheckedAt + } + } + healthyG, unhealthyG := map[string]struct{}{}, map[string]struct{}{} + for _, node := range order { + ok, kerr := known.eligible(node) + if kerr != nil { + return 0, 0, time.Time{}, kerr + } + if !ok { + continue + } + v := per[node] + g := known.groupOf(node) + switch { + case v.bad: + unhealthyG[g] = struct{}{} + case !v.newest.IsZero(): + healthyG[g] = struct{}{} + if v.newest.After(newest) { + newest = v.newest + } + } + } + return len(healthyG), len(unhealthyG), newest, nil +} + +// NetworkHealthyFiles labels releases for candidate lists from file-level +// evidence, so it works for a release this node only knows by its files and +// counts evidence published under any release ID. releases maps a release ID +// to the File IDs of its main (video) files. +// +// A release is labelled healthy when every one of its files is healthy under +// the same rule NetworkHealthy applies to a release: on some receiving network +// enough eligible publishers (distinct network prefixes on the open network) +// attest the file healthy with enough coverage, and on no network does +// unhealthy evidence reach the same threshold. Requiring every main file is the +// conservative reading of "the release is healthy": a release is only as +// available as its worst file, and a publisher that attested only some of its +// files has not vouched for the release. A release with no files, or with more +// than MaxLabelFilesPerRelease, is not labelled (it cannot be proven), and +// once MaxLabelFiles distinct files are collected the remaining releases are +// left unlabelled. The cost is one attestation lookup per receiving network +// for the whole call plus the batched publisher-standing lookups. Every +// requested release is present in the result; at most 200 are accepted. +func (a *Advisor) NetworkHealthyFiles(ctx context.Context, releases map[string][]string) (map[string]NetworkHealth, error) { + return a.NetworkHealthyFilesExcluding(ctx, "", releases) +} + +// NetworkHealthyFilesExcluding is NetworkHealthyFiles with every attestation +// of the publisher exclude (a node ID, 64 lower-case hex characters; empty +// excludes nobody) left out of the evidence, so a publisher's own attestation +// is not counted as independent evidence for its own offer. +func (a *Advisor) NetworkHealthyFilesExcluding(ctx context.Context, exclude string, releases map[string][]string) (map[string]NetworkHealth, error) { + if len(releases) > maxReleaseIDs { + return nil, ErrTooManyIDs + } + out := make(map[string]NetworkHealth, len(releases)) + rels := make([]string, 0, len(releases)) + for rel := range releases { + if !validID(rel) { + return nil, errors.New("hints: release ids must be 64 lower-case hex characters") + } + for _, f := range releases[rel] { + if !validID(f) { + return nil, errors.New("hints: file ids must be 64 lower-case hex characters") + } + } + out[rel] = NetworkHealth{} + rels = append(rels, rel) + } + sort.Strings(rels) + + // Decide which releases can be looked at, within the file budget. + files := map[string][]string{} + all := map[string]struct{}{} + for _, rel := range rels { + fs := uniqueStrings(releases[rel]) + if len(fs) == 0 || len(fs) > MaxLabelFilesPerRelease { + continue + } + fresh := 0 + for _, f := range fs { + if _, in := all[f]; !in { + fresh++ + } + } + if len(all)+fresh > MaxLabelFiles { + continue + } + for _, f := range fs { + all[f] = struct{}{} + } + files[rel] = fs + } + if len(files) == 0 { + return out, nil + } + ids := make([]string, 0, len(all)) + for f := range all { + ids = append(ids, f) + } + sort.Strings(ids) + + now := a.cfg.Now() + notBefore := now.Add(-records.AttestationValidity) + type fileState struct { + healthyOK, vetoed bool + healthy, bad int + newest time.Time + } + state := make(map[string]*fileState, len(ids)) + for _, f := range ids { + state[f] = &fileState{} + } + for _, n := range a.receiving() { + atts, err := a.cfg.Store.AttestationsForFiles(ctx, n.ID.String(), ids, notBefore) + if err != nil { + return nil, fmt.Errorf("hints: attestations: %w", err) + } + byFile := map[string][]database.PeernetAttestation{} + for _, att := range withoutPublisher(atts, exclude) { + if _, wanted := all[att.FileID]; wanted { + byFile[att.FileID] = append(byFile[att.FileID], att) + } + } + known := newKnownCache(a, ctx, n, now) + var pubs []string + seen := map[string]struct{}{} + for _, f := range ids { + for _, p := range publishersOf(byFile[f], a, now) { + if _, dup := seen[p]; !dup { + seen[p] = struct{}{} + pubs = append(pubs, p) + } + } + } + if err := known.prefetch(pubs); err != nil { + return nil, err + } + for _, f := range ids { + hE, uE, newest, err := a.tallyWith(known, byFile[f], now) + if err != nil { + return nil, err + } + st := state[f] + st.healthy += hE + st.bad += uE + if newest.After(st.newest) { + st.newest = newest + } + if hE >= a.threshold(n) { + st.healthyOK = true + } + if uE >= a.threshold(n) { + st.vetoed = true + } + } + } + for rel, fs := range files { + h := NetworkHealth{Healthy: true} + for i, f := range fs { + st := state[f] + if !st.healthyOK || st.vetoed { + h.Healthy = false + } + // The weakest file describes the release. + if i == 0 || st.healthy < h.HealthyPublishers { + h.HealthyPublishers = st.healthy + } + if st.bad > h.UnhealthyPublishers { + h.UnhealthyPublishers = st.bad + } + if i == 0 || st.newest.Before(h.Newest) { + h.Newest = st.newest + } + } + out[rel] = h + } + return out, nil +} + +func uniqueStrings(in []string) []string { + seen := make(map[string]struct{}, len(in)) + out := make([]string, 0, len(in)) + for _, s := range in { + if _, dup := seen[s]; !dup { + seen[s] = struct{}{} + out = append(out, s) + } + } + return out +} + +// withoutPublisher drops the attestations of the publisher exclude (none when +// empty). The input is not modified. +func withoutPublisher(atts []database.PeernetAttestation, exclude string) []database.PeernetAttestation { + if exclude == "" { + return atts + } + out := make([]database.PeernetAttestation, 0, len(atts)) + for _, att := range atts { + if att.PublisherNodeID != exclude { + out = append(out, att) + } + } + return out +} diff --git a/internal/peernet/publish/bundle_test.go b/internal/peernet/publish/bundle_test.go new file mode 100644 index 000000000..d7f200a68 --- /dev/null +++ b/internal/peernet/publish/bundle_test.go @@ -0,0 +1,248 @@ +package publish + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/peernet/records" +) + +var wantBundle = records.BundleID{0xb, 0xd, 0x1} + +type fakeBundles struct { + calls int + dec BundleDecision +} + +func (f *fakeBundles) BundleFor(context.Context, records.ReleaseID, ReleaseInfo) BundleDecision { + f.calls++ + return f.dec +} + +// releaseBundle returns the bundle id of the n-th release record sunk. +func releaseBundle(t *testing.T, e *env, n int) records.BundleID { + t.Helper() + seen := 0 + for _, s := range e.sink.got { + env, err := records.Peek(s.enc) + if err != nil { + t.Fatal(err) + } + if env.Kind != records.KindRelease { + continue + } + if seen == n { + r, err := records.DecodeReleaseRecord(s.enc) + if err != nil { + t.Fatal(err) + } + return r.BundleID + } + seen++ + } + t.Fatalf("no release record #%d", n) + return records.BundleID{} +} + +func lastRelease(t *testing.T, e *env) State { + t.Helper() + st, ok, err := e.store.LastPublished(context.Background(), openNet, records.KindRelease, relID, records.FileID{}) + if err != nil || !ok { + t.Fatalf("no release state: %v %v", ok, err) + } + return st +} + +func TestBundleIDIsAnnouncedOnlyAtLevelThree(t *testing.T) { + for _, tc := range []struct { + level int + want records.BundleID + wantCalls int + }{{2, records.BundleID{}, 0}, {3, wantBundle, 1}} { + e := newEnv(t) + fb := &fakeBundles{dec: BundleDecision{ID: wantBundle}} + e.cfg.Bundles = fb + e.nets = []Network{{ID: openNet, Level: tc.level}} + if _, err := e.publisher(t).Publish(context.Background(), healthy(), info()); err != nil { + t.Fatal(err) + } + if got := releaseBundle(t, e, 0); got != tc.want { + t.Errorf("level %d: bundle id = %x, want %x", tc.level, got, tc.want) + } + if fb.calls != tc.wantCalls { + t.Errorf("level %d: bundle source called %d times, want %d", tc.level, fb.calls, tc.wantCalls) + } + st := lastRelease(t, e) + if st.BundleID != tc.want || st.BundleLevel != tc.level { + t.Errorf("level %d: state bundle %x level %d", tc.level, st.BundleID, st.BundleLevel) + } + } +} + +func TestLevelChangeSupersedesTheRecord(t *testing.T) { + e := newEnv(t) + fb := &fakeBundles{dec: BundleDecision{ID: wantBundle}} + e.cfg.Bundles = fb + p := e.publisher(t) + ctx := context.Background() + if _, err := p.Publish(ctx, healthy(), info()); err != nil { + t.Fatal(err) + } + // Nothing changed: not due, and the bundle source is not consulted again. + if o := p.ReannounceRelease(ctx, openNet, relID, info()); o.Skip != SkipNotDue { + t.Fatalf("unchanged: %+v", o) + } + // Level 2 -> 3: the record is announced again, now with the bundle. + e.nets = []Network{{ID: openNet, Level: 3}} + if o := p.ReannounceRelease(ctx, openNet, relID, info()); o.Skip != SkipNone || o.Err != nil { + t.Fatalf("2->3: %+v", o) + } + if got := releaseBundle(t, e, 1); got != wantBundle { + t.Fatalf("2->3 record bundle = %x", got) + } + if o := p.ReannounceRelease(ctx, openNet, relID, info()); o.Skip != SkipNotDue { + t.Fatalf("settled at 3: %+v", o) + } + // Level 3 -> 2: superseded by a record without one. + e.nets = []Network{{ID: openNet, Level: 2}} + e.now = e.now.Add(time.Second) + if o := p.ReannounceRelease(ctx, openNet, relID, info()); o.Skip != SkipNone || o.Err != nil { + t.Fatalf("3->2: %+v", o) + } + if got := releaseBundle(t, e, 2); got != (records.BundleID{}) { + t.Fatalf("3->2 record still carries a bundle id %x", got) + } + if st := lastRelease(t, e); st.BundleID != (records.BundleID{}) || st.BundleLevel != 2 { + t.Fatalf("state after drop: %+v", st) + } + if fb.calls != 1 { + t.Fatalf("bundle source called %d times, want exactly once", fb.calls) + } +} + +func TestNotShareableReleaseStillPublishesItsRecordAndIsNotRetried(t *testing.T) { + e := newEnv(t) + fb := &fakeBundles{dec: BundleDecision{Skip: "rclone_encrypted"}} + e.cfg.Bundles = fb + e.nets = []Network{{ID: openNet, Level: 3}} + p := e.publisher(t) + outs, err := p.Publish(context.Background(), healthy(), info()) + if err != nil { + t.Fatal(err) + } + if published(outs, records.KindRelease) != 1 { + t.Fatalf("release record not published: %+v", outs) + } + if releaseBundle(t, e, 0) != (records.BundleID{}) { + t.Fatal("a not shareable release announced a bundle") + } + if st := lastRelease(t, e); st.BundleLevel != 3 || st.BundleID != (records.BundleID{}) { + t.Fatalf("the decision must be remembered: %+v", st) + } + if o := p.ReannounceRelease(context.Background(), openNet, relID, info()); o.Skip != SkipNotDue || fb.calls != 1 { + t.Fatalf("retried a settled decision: %+v calls=%d", o, fb.calls) + } +} + +func TestTransientBundleFailureSendsNothingAndKeepsState(t *testing.T) { + e := newEnv(t) + fb := &fakeBundles{dec: BundleDecision{ID: wantBundle}} + e.cfg.Bundles = fb + e.nets = []Network{{ID: openNet, Level: 3}} + p := e.publisher(t) + ctx := context.Background() + if _, err := p.Publish(ctx, healthy(), info()); err != nil { + t.Fatal(err) + } + before := lastRelease(t, e) + sent := len(e.sink.got) + + // The announced record had a bundle: a busy builder must not supersede it + // with a bundle-less one, and must not churn while the failure lasts. + fb.dec = BundleDecision{Err: errors.New("builder busy")} + e.now = e.now.Add(DefaultRecordRefresh + time.Hour) // the record is due for refresh + for range 3 { + o := p.ReannounceRelease(ctx, openNet, relID, info()) + if o.BundleErr == nil || o.Skip == SkipNone || o.Err != nil { + t.Fatalf("busy: %+v", o) + } + } + if len(e.sink.got) != sent { + t.Fatalf("records sent during failure: %d, want %d", len(e.sink.got)-sent, 0) + } + if got := lastRelease(t, e); got != before { + t.Fatalf("state changed during failure: %+v -> %+v", before, got) + } + + // Recovery: exactly one record, with the bundle. + fb.dec = BundleDecision{ID: wantBundle} + if o := p.ReannounceRelease(ctx, openNet, relID, info()); o.Skip != SkipNone || o.Err != nil || o.BundleErr != nil { + t.Fatalf("retry: %+v", o) + } + if len(e.sink.got) != sent+1 || releaseBundle(t, e, 1) != wantBundle { + t.Fatalf("recovery sent %d records", len(e.sink.got)-sent) + } +} + +func TestTransientBundleFailureOnFirstPublishIsRetriedWithoutChurn(t *testing.T) { + e := newEnv(t) + fb := &fakeBundles{dec: BundleDecision{Err: errors.New("builder busy")}} + e.cfg.Bundles = fb + e.nets = []Network{{ID: openNet, Level: 3}} + p := e.publisher(t) + ctx := context.Background() + for range 3 { + outs, err := p.Publish(ctx, healthy(), info()) + if err != nil { + t.Fatal(err) + } + if published(outs, records.KindRelease) != 0 { + t.Fatalf("release published during failure: %+v", outs) + } + } + for _, s := range e.sink.got { + if env, _ := records.Peek(s.enc); env.Kind == records.KindRelease { + t.Fatal("release record sent during failure") + } + } + fb.dec = BundleDecision{ID: wantBundle} + if o := p.ReannounceRelease(ctx, openNet, relID, info()); o.Skip != SkipNone || o.Err != nil { + t.Fatalf("retry: %+v", o) + } + if releaseBundle(t, e, 0) != wantBundle { + t.Fatal("retry did not announce the bundle") + } +} + +func TestMoreFileNoticesOfAPublishedReleaseDoNotRebuild(t *testing.T) { + e := newEnv(t) + fb := &fakeBundles{dec: BundleDecision{ID: wantBundle}} + e.cfg.Bundles = fb + e.nets = []Network{{ID: openNet, Level: 3}} + p := e.publisher(t) + for range 3 { + if _, err := p.Publish(context.Background(), healthy(), info()); err != nil { + t.Fatal(err) + } + } + if fb.calls != 1 { + t.Fatalf("bundle source called %d times for one release", fb.calls) + } +} + +func TestExclusionsCoverOrigin(t *testing.T) { + ex := Exclusions{Categories: []string{" Movies "}, Indexers: []string{"IdxOne", "unknown"}} + for _, tc := range []struct { + cat, idx string + want bool + }{ + {"movies", "", true}, {"tv", "idxone", true}, {"tv", "other", false}, + {"", "", false}, {"tv", "unknown", false}, + } { + if got := ex.CoversOrigin(tc.cat, tc.idx); got != tc.want { + t.Errorf("%q/%q = %v, want %v", tc.cat, tc.idx, got, tc.want) + } + } +} diff --git a/internal/peernet/publish/plan.go b/internal/peernet/publish/plan.go index 15ff763c2..979cd8929 100644 --- a/internal/peernet/publish/plan.go +++ b/internal/peernet/publish/plan.go @@ -50,6 +50,12 @@ type State struct { // are never put in a record. OriginCategory string OriginIndexer string + // BundleID is the Bundle ID the release record announced (zero: none) and + // BundleLevel the exposure level the record was published at. Release + // records only. A transient bundle failure stores nothing: the release is + // skipped and retried. + BundleID records.BundleID + BundleLevel int } // DueRelease reports whether a release record should be published now: never diff --git a/internal/peernet/publish/publish.go b/internal/peernet/publish/publish.go index a726b1567..9a4dce88f 100644 --- a/internal/peernet/publish/publish.go +++ b/internal/peernet/publish/publish.go @@ -104,12 +104,38 @@ type ReleaseInfo struct { Media records.MediaIDs Category string Indexer string + // ReleaseScheme says how the Release ID was derived: records.SchemeNZB (also + // the zero value) or records.SchemeFiles. It goes into both record types. + ReleaseScheme uint8 // Secrets are the import's passwords (archive password, password spelled // in the NZB name). They are used only to refuse a title that contains // one: never stored, logged or published. Secrets []string } +// BundleMinLevel is the lowest exposure level that announces a bundle. +const BundleMinLevel = 3 + +// BundleDecision is what a BundleSource says about one release. +type BundleDecision struct { + // ID is the Bundle ID to announce; zero when the release has no bundle. + ID records.BundleID + // Skip names the permanent reason a release cannot be shared as a bundle + // (the source counts it). The record is published without a Bundle ID and + // the decision is remembered. + Skip string + // Err is a transient failure. The record is published without a Bundle ID + // and the decision is not remembered, so it is made again soon. + Err error +} + +// BundleSource builds the bundle of a release and returns its ID. It is +// called only for a network at level BundleMinLevel or higher, and only when a +// release record is about to be published. +type BundleSource interface { + BundleFor(ctx context.Context, rel records.ReleaseID, info ReleaseInfo) BundleDecision +} + // Network is one network and this node's exposure level on it. type Network struct { ID netid.NetworkID @@ -182,6 +208,9 @@ type Config struct { Networks func() []Network Exclusions func() Exclusions Now func() time.Time + // Bundles supplies Bundle IDs for networks at level 3. Nil: no record + // carries one. + Bundles BundleSource Unknown UnknownPolicy // PublishUnhealthy opts in to unhealthy attestations. Default off. @@ -207,6 +236,7 @@ const ( SkipNoBackbone SkipReason = "no_recognised_backbone" SkipStale SkipReason = "check_too_old" SkipNotDue SkipReason = "not_due" + SkipBundleUnavailable SkipReason = "bundle_unavailable" SkipTitleHasPassword SkipReason = "title_contains_password" ) @@ -217,6 +247,11 @@ type Outcome struct { Kind records.Kind Skip SkipReason Err error + // BundleErr is a transient failure to build the bundle of a release + // record. The release is skipped (SkipBundleUnavailable) and its stored + // state kept, so a record that had a bundle is never superseded by one + // without it; the next pass retries. + BundleErr error } // Publisher builds and publishes records. It is safe for concurrent use as @@ -279,7 +314,7 @@ func (p *Publisher) Publish(ctx context.Context, res CheckResult, info ReleaseIn if err := ctx.Err(); err != nil { return outs, errors.Join(append(errs, err)...) } - for _, o := range job.run(ctx, n.ID) { + for _, o := range job.run(ctx, n) { outs = append(outs, o) if o.Err != nil { errs = append(errs, o.Err) @@ -309,10 +344,10 @@ func (p *Publisher) ReannounceRelease(ctx context.Context, nw netid.NetworkID, r o.Err = err return o } - allowed := false + allowed, level := false, 0 for _, n := range p.cfg.Networks() { if n.ID == nw && n.Level >= MinLevel { - allowed = true + allowed, level = true, n.Level break } } @@ -325,11 +360,13 @@ func (p *Publisher) ReannounceRelease(ctx context.Context, nw netid.NetworkID, r return o } j := &job{p: p, res: CheckResult{ReleaseID: rel}, info: info, now: now} - return j.release(ctx, nw) + return j.release(ctx, nw, level) } func validate(res CheckResult, info ReleaseInfo) error { switch { + case info.ReleaseScheme > records.SchemeFiles: + return fmt.Errorf("%w: release scheme", ErrInvalidInput) case res.ReleaseID == records.ReleaseID{}: return fmt.Errorf("%w: zero release id", ErrInvalidInput) case res.FileID == records.FileID{}: @@ -452,15 +489,34 @@ func (j *job) backbones(ctx context.Context) []string { return j.backbone } -func (j *job) run(ctx context.Context, nw netid.NetworkID) []Outcome { +func (j *job) run(ctx context.Context, n Network) []Outcome { var outs []Outcome if j.res.Verdict == VerdictHealthy { - outs = append(outs, j.release(ctx, nw)) + outs = append(outs, j.release(ctx, n.ID, n.Level)) } - return append(outs, j.attestation(ctx, nw)) + return append(outs, j.attestation(ctx, n.ID)) } -func (j *job) release(ctx context.Context, nw netid.NetworkID) Outcome { +// bundleFor decides the bundle part of a release record for a network at the +// given level: the Bundle ID to announce and the bundle level to remember. +// Below level 3 (or without a source) the record carries no bundle and the +// remembered level is the network's. A transient failure returns an error and +// the caller publishes nothing for this pass. +func (j *job) bundleFor(ctx context.Context, level int) (id records.BundleID, remembered int, err error) { + if level < BundleMinLevel || j.p.cfg.Bundles == nil { + return records.BundleID{}, level, nil + } + d := j.p.cfg.Bundles.BundleFor(ctx, j.res.ReleaseID, j.info) + switch { + case d.Err != nil: + return records.BundleID{}, 0, d.Err + case d.Skip == "": + return d.ID, level, nil + } + return records.BundleID{}, level, nil +} + +func (j *job) release(ctx context.Context, nw netid.NetworkID, level int) Outcome { o := Outcome{Network: nw, Kind: records.KindRelease} var zero records.FileID last, ok, err := j.p.cfg.Store.LastPublished(ctx, nw, records.KindRelease, j.res.ReleaseID, zero) @@ -472,16 +528,24 @@ func (j *job) release(ctx context.Context, nw netid.NetworkID) Outcome { if ok { lp = &last } - if !j.p.cfg.Plan.DueRelease(lp, j.now) { + // A level change (below 3 to 3 or back) supersedes the record at once: it + // is the bundle part of the record that differs. + levelChanged := lp != nil && (lp.BundleLevel >= BundleMinLevel) != (level >= BundleMinLevel) + if !j.p.cfg.Plan.DueRelease(lp, j.now) && !levelChanged { o.Skip = SkipNotDue return o } - enc, err := j.buildRelease(nw) + bundle, remembered, bundleErr := j.bundleFor(ctx, level) + if bundleErr != nil { + o.BundleErr, o.Skip = bundleErr, SkipBundleUnavailable + return o + } + enc, err := j.buildRelease(nw, bundle) if err != nil { o.Err = err return o } - o.Err = j.send(ctx, nw, records.KindRelease, zero, enc, State{PublishedAt: j.now}) + o.Err = j.send(ctx, nw, records.KindRelease, zero, enc, State{PublishedAt: j.now, BundleID: bundle, BundleLevel: remembered}) return o } @@ -530,15 +594,16 @@ func (j *job) attestation(ctx context.Context, nw netid.NetworkID) Outcome { return o } -func (j *job) buildRelease(nw netid.NetworkID) ([]byte, error) { +func (j *job) buildRelease(nw netid.NetworkID, bundle records.BundleID) ([]byte, error) { in := records.ReleaseRecord{ + BundleID: bundle, Title: releaseTitle(j.info.Title), TotalSize: j.info.TotalSize, FileCount: j.info.FileCount, PostDate: j.info.PostDate, Media: j.info.Media, } - in.ReleaseID = j.res.ReleaseID + in.ReleaseID, in.ReleaseScheme = j.res.ReleaseID, j.info.ReleaseScheme enc, err := j.p.cfg.Signer.BuildRelease(nw, in, j.now) if err != nil && errors.Is(err, records.ErrInvalidField) { // Optional data from a local source is dropped rather than losing the @@ -562,7 +627,7 @@ func (j *job) buildAttestation(nw netid.NetworkID, checked time.Time, labels []s MediaHeaderVerified: j.res.MediaHeaderVerified && j.res.Verdict == VerdictHealthy, Backbones: labels, } - in.ReleaseID = j.res.ReleaseID + in.ReleaseID, in.ReleaseScheme = j.res.ReleaseID, j.info.ReleaseScheme enc, err := j.p.cfg.Signer.BuildAttestation(nw, in, j.now) if err != nil { return nil, fmt.Errorf("publish: build attestation: %w", err) @@ -643,3 +708,14 @@ func titleLeaks(title string, secrets []string) bool { } return false } + +// CoversOrigin reports whether an origin tag (as kept with the publish state: +// lower-cased category and indexer, either possibly empty) falls under an +// exclusion. An empty tag and the indexer "unknown" match nothing. +func (e Exclusions) CoversOrigin(category, indexer string) bool { + category, indexer = normName(category), normName(indexer) + if category != "" && listed(e.Categories, category) { + return true + } + return indexer != "" && indexer != "unknown" && listed(e.Indexers, indexer) +} diff --git a/internal/peernet/publish/scheme_test.go b/internal/peernet/publish/scheme_test.go new file mode 100644 index 000000000..ef1ecaedc --- /dev/null +++ b/internal/peernet/publish/scheme_test.go @@ -0,0 +1,74 @@ +package publish + +import ( + "context" + "testing" + + "github.com/kipsilabs/altmount/internal/peernet/records" +) + +func schemes(t *testing.T, sink *fakeSink) map[records.Kind]uint8 { + t.Helper() + out := map[records.Kind]uint8{} + for _, s := range sink.got { + env, err := records.Peek(s.enc) + if err != nil { + t.Fatal(err) + } + out[env.Kind] = env.ReleaseScheme + } + return out +} + +// A release identified by its files says so in both records, so a receiver +// never mistakes it for an NZB-derived ID. +func TestPublishCarriesReleaseScheme(t *testing.T) { + for _, tc := range []struct { + name string + in uint8 + want uint8 + }{{"default is nzb", 0, records.SchemeNZB}, {"nzb", records.SchemeNZB, records.SchemeNZB}, {"files", records.SchemeFiles, records.SchemeFiles}} { + t.Run(tc.name, func(t *testing.T) { + e := newEnv(t) + inf := info() + inf.ReleaseScheme = tc.in + if _, err := e.publisher(t).Publish(context.Background(), healthy(), inf); err != nil { + t.Fatal(err) + } + got := schemes(t, e.sink) + if len(got) != 2 { + t.Fatalf("published kinds %v", got) + } + for k, s := range got { + if s != tc.want { + t.Errorf("%v record has scheme %d, want %d", k, s, tc.want) + } + } + }) + } +} + +func TestReannounceCarriesReleaseScheme(t *testing.T) { + e := newEnv(t) + inf := info() + inf.ReleaseScheme = records.SchemeFiles + o := e.publisher(t).ReannounceRelease(context.Background(), openNet, relID, inf) + if o.Err != nil || o.Skip != SkipNone { + t.Fatalf("outcome %+v", o) + } + if got := schemes(t, e.sink)[records.KindRelease]; got != records.SchemeFiles { + t.Fatalf("re-announced record has scheme %d, want files", got) + } +} + +func TestPublishRejectsUnknownScheme(t *testing.T) { + e := newEnv(t) + inf := info() + inf.ReleaseScheme = 7 + if _, err := e.publisher(t).Publish(context.Background(), healthy(), inf); err == nil { + t.Fatal("an unknown scheme must not become a record") + } + if len(e.sink.got) != 0 { + t.Fatal("nothing may be sunk") + } +} diff --git a/internal/peernet/records/attestation.go b/internal/peernet/records/attestation.go index 45b663404..10c5bd002 100644 --- a/internal/peernet/records/attestation.go +++ b/internal/peernet/records/attestation.go @@ -99,7 +99,7 @@ func (a *HealthAttestation) Verify(now time.Time, arrivedIn netid.NetworkID) err // are ignored; expiry is AttestationValidity after creation. Backbone labels // are trimmed, lower-cased, sorted and de-duplicated on a copy. func BuildHealthAttestation(key *identity.Key, nw netid.NetworkID, in HealthAttestation, now time.Time) (*HealthAttestation, []byte, error) { - h, err := newHeader(key, nw, in.ReleaseID, now, AttestationValidity) + h, err := newHeader(key, nw, in.ReleaseID, in.ReleaseScheme, now, AttestationValidity) if err != nil { return nil, nil, err } diff --git a/internal/peernet/records/doc.go b/internal/peernet/records/doc.go index 9038a1dcf..2e8f6e932 100644 --- a/internal/peernet/records/doc.go +++ b/internal/peernet/records/doc.go @@ -29,6 +29,16 @@ // 6 publisher_key bytes 32 (Ed25519) // 7 created_at varint // 8 expires_at varint +// 9 release_scheme varint optional; absent means 1 ("nzb": the +// release_id hashes the NZB segment list); +// present must be exactly 2 ("files": the +// release_id is derived from File IDs, see +// contentid.ReleaseIDFromFiles). 1 and every +// other value are rejected. It is part of the +// signed body. Records without it are +// byte-identical to those signed before it +// existed. Nodes that predate it reject a +// record that carries it (unknown field). // // Release record: // diff --git a/internal/peernet/records/envelope.go b/internal/peernet/records/envelope.go index 59461f41d..09ac8d9ba 100644 --- a/internal/peernet/records/envelope.go +++ b/internal/peernet/records/envelope.go @@ -10,14 +10,16 @@ import ( // Envelope is the indexable summary of an encoded record. type Envelope struct { - Kind Kind - NetworkID netid.NetworkID - ReleaseID ReleaseID - FileID FileID // zero for release records - PublisherID identity.NodeID - CreatedAt time.Time - ExpiresAt time.Time - ContentHash [32]byte + Kind Kind + NetworkID netid.NetworkID + ReleaseID ReleaseID + FileID FileID // zero for release records + // ReleaseScheme is SchemeNZB or SchemeFiles (an absent field is SchemeNZB). + ReleaseScheme uint8 + PublisherID identity.NodeID + CreatedAt time.Time + ExpiresAt time.Time + ContentHash [32]byte } // Peek strictly decodes an encoded record of either kind and returns its @@ -57,7 +59,7 @@ func Peek(b []byte) (Envelope, error) { func envelopeOf(k Kind, h *Header, file FileID, b []byte) Envelope { return Envelope{ Kind: k, NetworkID: h.NetworkID, ReleaseID: h.ReleaseID, FileID: file, - PublisherID: h.PublisherID, CreatedAt: h.CreatedAt, ExpiresAt: h.ExpiresAt, + ReleaseScheme: h.Scheme(), PublisherID: h.PublisherID, CreatedAt: h.CreatedAt, ExpiresAt: h.ExpiresAt, ContentHash: ContentHash(b), } } diff --git a/internal/peernet/records/fuzz_test.go b/internal/peernet/records/fuzz_test.go index 808847e83..09508e068 100644 --- a/internal/peernet/records/fuzz_test.go +++ b/internal/peernet/records/fuzz_test.go @@ -4,6 +4,7 @@ import ( "bytes" "testing" + "github.com/kipsilabs/altmount/internal/peernet/identity" "github.com/kipsilabs/altmount/internal/peernet/netid" ) @@ -11,8 +12,14 @@ func FuzzDecodeReleaseRecord(f *testing.F) { k := testKey(f) _, enc := mustRelease(f, k, netid.Open()) _, att := mustAttestation(f, k, netid.Open()) + fileEnc, fileAtt := filesSchemeSeeds(f, k) f.Add(enc) f.Add(att) + f.Add(fileEnc) + f.Add(fileAtt) + f.Add(withSchemeField(f, enc, 1)) + f.Add(withSchemeField(f, enc, 2)) + f.Add(withSchemeField(f, att, 3)) f.Add([]byte{}) f.Add([]byte{0x08, 0x01}) f.Add(enc[:len(enc)/2]) @@ -39,8 +46,14 @@ func FuzzDecodeHealthAttestation(f *testing.F) { k := testKey(f) _, enc := mustRelease(f, k, netid.Open()) _, att := mustAttestation(f, k, netid.Open()) + fileEnc, fileAtt := filesSchemeSeeds(f, k) f.Add(att) f.Add(enc) + f.Add(fileEnc) + f.Add(fileAtt) + f.Add(withSchemeField(f, att, 1)) + f.Add(withSchemeField(f, att, 2)) + f.Add(withSchemeField(f, enc, 0)) f.Add([]byte{}) f.Add([]byte{0x08, 0x01, 0x10, 0x02}) f.Add(att[:len(att)/2]) @@ -62,3 +75,21 @@ func FuzzDecodeHealthAttestation(f *testing.F) { } }) } + +// filesSchemeSeeds returns signed records that carry release_scheme = 2. +func filesSchemeSeeds(tb testing.TB, k *identity.Key) (release, attestation []byte) { + tb.Helper() + rin := releaseInput() + rin.ReleaseScheme = SchemeFiles + _, release, err := BuildReleaseRecord(k, netid.Open(), rin, testNow) + if err != nil { + tb.Fatal(err) + } + ain := attestationInput() + ain.ReleaseScheme = SchemeFiles + _, attestation, err = BuildHealthAttestation(k, netid.Open(), ain, testNow) + if err != nil { + tb.Fatal(err) + } + return release, attestation +} diff --git a/internal/peernet/records/limits.go b/internal/peernet/records/limits.go index 496a4d256..a4185be57 100644 --- a/internal/peernet/records/limits.go +++ b/internal/peernet/records/limits.go @@ -7,6 +7,13 @@ import ( // Limits. Several are design calls the owner may tune. const ( + // SchemeNZB: the Release ID is the hash of the release's NZB segment list. + // It is the meaning of an absent release_scheme field. + SchemeNZB uint8 = 1 + // SchemeFiles: the Release ID is derived from the release's File IDs (see + // contentid.ReleaseIDFromFiles). It is carried as release_scheme = 2. + SchemeFiles uint8 = 2 + // SchemaVersion is the only version this package reads and writes. SchemaVersion = 1 diff --git a/internal/peernet/records/release.go b/internal/peernet/records/release.go index 6bf52b254..eabcfda37 100644 --- a/internal/peernet/records/release.go +++ b/internal/peernet/records/release.go @@ -115,7 +115,7 @@ func (r *ReleaseRecord) Verify(now time.Time, arrivedIn netid.NetworkID) error { // title is cleaned; other invalid input is an error. It returns the record and // its encoding. func BuildReleaseRecord(key *identity.Key, nw netid.NetworkID, in ReleaseRecord, now time.Time) (*ReleaseRecord, []byte, error) { - h, err := newHeader(key, nw, in.ReleaseID, now, RecordLifetime) + h, err := newHeader(key, nw, in.ReleaseID, in.ReleaseScheme, now, RecordLifetime) if err != nil { return nil, nil, err } diff --git a/internal/peernet/records/scheme_test.go b/internal/peernet/records/scheme_test.go new file mode 100644 index 000000000..0bff6c31e --- /dev/null +++ b/internal/peernet/records/scheme_test.go @@ -0,0 +1,192 @@ +package records + +import ( + "bytes" + "errors" + "testing" + + "google.golang.org/protobuf/encoding/protowire" + + "github.com/kipsilabs/altmount/internal/peernet/netid" +) + +// fieldNumbers lists the field numbers of an encoded record in order. +func fieldNumbers(t *testing.T, b []byte) []protowire.Number { + t.Helper() + fs, err := parseFields(b, fBackbone) + if err != nil { + t.Fatal(err) + } + out := make([]protowire.Number, len(fs)) + for i, f := range fs { + out[i] = f.num + } + return out +} + +func has(nums []protowire.Number, n protowire.Number) bool { + for _, x := range nums { + if x == n { + return true + } + } + return false +} + +// withSchemeField inserts field 9 with value v after field 8 and re-signs +// nothing: used to build hostile encodings. +func withSchemeField(t testing.TB, enc []byte, v uint64) []byte { + t.Helper() + var out []byte + rest := enc + inserted := false + for len(rest) > 0 { + num, typ, n := protowire.ConsumeTag(rest) + m := protowire.ConsumeFieldValue(num, typ, rest[n:]) + if num > fExpires && !inserted { + out = protowire.AppendVarint(protowire.AppendTag(out, 9, protowire.VarintType), v) + inserted = true + } + out = append(out, rest[:n+m]...) + rest = rest[n+m:] + } + return out +} + +func TestScheme_AbsentIsNZBAndEncodingUnchanged(t *testing.T) { + k := testKey(t) + nw := netid.Open() + rec, enc := mustRelease(t, k, nw) + if has(fieldNumbers(t, enc), 9) { + t.Fatal("nzb scheme must not add a field: records of the released encoding must stay byte-identical") + } + if rec.Scheme() != SchemeNZB { + t.Fatalf("scheme %d, want nzb", rec.Scheme()) + } + in := releaseInput() + in.ReleaseScheme = SchemeNZB + _, explicit, err := BuildReleaseRecord(k, nw, in, testNow) + if err != nil { + t.Fatal(err) + } + if has(fieldNumbers(t, explicit), 9) { + t.Fatal("explicit nzb scheme must still be absent from the encoding") + } + got, err := DecodeReleaseRecord(enc) + if err != nil || got.Scheme() != SchemeNZB { + t.Fatalf("decode: %v scheme %v", err, got) + } + _, att := mustAttestation(t, k, nw) + if has(fieldNumbers(t, att), 9) { + t.Fatal("attestation default must not carry field 9") + } +} + +func TestScheme_FilesRoundTripBothTypes(t *testing.T) { + k := testKey(t) + nw := netid.Open() + rin := releaseInput() + rin.ReleaseScheme = SchemeFiles + rec, enc, err := BuildReleaseRecord(k, nw, rin, testNow) + if err != nil { + t.Fatal(err) + } + if !has(fieldNumbers(t, enc), 9) { + t.Fatal("files scheme must be encoded in field 9") + } + got, err := DecodeReleaseRecord(enc) + if err != nil { + t.Fatal(err) + } + if got.Scheme() != SchemeFiles || rec.Scheme() != SchemeFiles { + t.Fatalf("scheme %d / %d", got.Scheme(), rec.Scheme()) + } + if err := got.Verify(testNow, nw); err != nil { + t.Fatal(err) + } + if again, err := got.Encode(); err != nil || !bytes.Equal(again, enc) { + t.Fatalf("re-encode differs: %v", err) + } + if env, err := Peek(enc); err != nil || env.ReleaseScheme != SchemeFiles { + t.Fatalf("peek: %v %+v", err, env) + } + + ain := attestationInput() + ain.ReleaseScheme = SchemeFiles + _, aenc, err := BuildHealthAttestation(k, nw, ain, testNow) + if err != nil { + t.Fatal(err) + } + a, err := DecodeHealthAttestation(aenc) + if err != nil || a.Scheme() != SchemeFiles { + t.Fatalf("attestation: %v", err) + } + if err := a.Verify(testNow, nw); err != nil { + t.Fatal(err) + } + if again, err := a.Encode(); err != nil || !bytes.Equal(again, aenc) { + t.Fatalf("attestation re-encode differs: %v", err) + } +} + +func TestScheme_AnyOtherValueRejected(t *testing.T) { + k := testKey(t) + nw := netid.Open() + _, enc := mustRelease(t, k, nw) + _, att := mustAttestation(t, k, nw) + for _, v := range []uint64{0, 1, 3, 255, 1 << 40} { + if _, err := DecodeReleaseRecord(withSchemeField(t, enc, v)); err == nil { + t.Fatalf("release record with scheme %d accepted", v) + } + if _, err := DecodeHealthAttestation(withSchemeField(t, att, v)); err == nil { + t.Fatalf("attestation with scheme %d accepted", v) + } + } + bad := releaseInput() + bad.ReleaseScheme = 3 + if _, _, err := BuildReleaseRecord(k, nw, bad, testNow); !errors.Is(err, ErrInvalidField) { + t.Fatalf("build with scheme 3: %v", err) + } + badA := attestationInput() + badA.ReleaseScheme = 7 + if _, _, err := BuildHealthAttestation(k, nw, badA, testNow); !errors.Is(err, ErrInvalidField) { + t.Fatalf("build attestation with scheme 7: %v", err) + } +} + +func TestScheme_CoveredBySignature(t *testing.T) { + k := testKey(t) + nw := netid.Open() + // A signed nzb-scheme record with field 9 = 2 spliced in must not verify. + _, enc := mustRelease(t, k, nw) + forged, err := DecodeReleaseRecord(withSchemeField(t, enc, 2)) + if err != nil { + t.Fatalf("spliced record should decode (canonical): %v", err) + } + if err := forged.Verify(testNow, nw); !errors.Is(err, ErrBadSignature) { + t.Fatalf("scheme spliced into a signed record verified: %v", err) + } + // And removing the field from a files-scheme record must not verify either. + in := releaseInput() + in.ReleaseScheme = SchemeFiles + rec, fenc, err := BuildReleaseRecord(k, nw, in, testNow) + if err != nil { + t.Fatal(err) + } + stripped := *rec + stripped.ReleaseScheme = 0 + stripped.Signature = rec.Signature + if err := stripped.Verify(testNow, nw); !errors.Is(err, ErrBadSignature) { + t.Fatalf("scheme stripped from a signed record verified: %v", err) + } + _ = fenc + // The same for attestations. + _, att := mustAttestation(t, k, nw) + fa, err := DecodeHealthAttestation(withSchemeField(t, att, 2)) + if err != nil { + t.Fatal(err) + } + if err := fa.Verify(testNow, nw); !errors.Is(err, ErrBadSignature) { + t.Fatalf("scheme spliced into a signed attestation verified: %v", err) + } +} diff --git a/internal/peernet/records/wire.go b/internal/peernet/records/wire.go index 10d432331..4ca930260 100644 --- a/internal/peernet/records/wire.go +++ b/internal/peernet/records/wire.go @@ -42,14 +42,18 @@ type ( // Field numbers (see the package documentation). const ( - fVersion protowire.Number = 1 - fKind protowire.Number = 2 - fNetwork protowire.Number = 3 - fRelease protowire.Number = 4 - fNodeID protowire.Number = 5 - fPubKey protowire.Number = 6 - fCreated protowire.Number = 7 - fExpires protowire.Number = 8 + fVersion protowire.Number = 1 + fKind protowire.Number = 2 + fNetwork protowire.Number = 3 + fRelease protowire.Number = 4 + fNodeID protowire.Number = 5 + fPubKey protowire.Number = 6 + fCreated protowire.Number = 7 + fExpires protowire.Number = 8 + // fScheme is the optional release_scheme header field. It sits between the + // common header (1-8) and the type-specific fields (10+), so a record + // without it is byte-identical to the encoding that predates it. + fScheme protowire.Number = 9 fSignature protowire.Number = 30 fBundle protowire.Number = 10 @@ -77,9 +81,12 @@ type Header struct { ReleaseID ReleaseID PublisherID identity.NodeID PublisherKey ed25519.PublicKey - CreatedAt time.Time - ExpiresAt time.Time - Signature []byte + // ReleaseScheme says how ReleaseID was derived: 0 or SchemeNZB (absent from + // the encoding) or SchemeFiles. See Scheme. + ReleaseScheme uint8 + CreatedAt time.Time + ExpiresAt time.Time + Signature []byte } // field is one parsed wire field. @@ -145,7 +152,7 @@ func validTime(t time.Time) bool { return u >= 1 && u <= maxUnixSeconds } -// appendHeader writes fields 1-8. +// appendHeader writes fields 1-8, and 9 only for a non-default scheme. func appendHeader(b []byte, k Kind, h *Header) []byte { b = appendVarint(b, fVersion, SchemaVersion) b = appendVarint(b, fKind, uint64(k)) @@ -154,7 +161,11 @@ func appendHeader(b []byte, k Kind, h *Header) []byte { b = appendBytes(b, fNodeID, h.PublisherID[:]) b = appendBytes(b, fPubKey, h.PublisherKey) b = appendVarint(b, fCreated, unixOf(h.CreatedAt)) - return appendVarint(b, fExpires, unixOf(h.ExpiresAt)) + b = appendVarint(b, fExpires, unixOf(h.ExpiresAt)) + if h.ReleaseScheme == SchemeFiles { + b = appendVarint(b, fScheme, uint64(SchemeFiles)) + } + return b } func appendSignature(b, sig []byte) []byte { return appendBytes(b, fSignature, sig) } @@ -224,6 +235,14 @@ func readHeaderField(f field, want Kind, h *Header) (bool, error) { h.CreatedAt, err = readTime(f, "created_at") case fExpires: h.ExpiresAt, err = readTime(f, "expires_at") + case fScheme: + // Present means exactly SchemeFiles: the default is expressed by + // leaving the field out, so any other value is non-canonical. + var v uint64 + if v, err = readVarint(f, ^uint64(0), "release_scheme"); err == nil && v != uint64(SchemeFiles) { + err = fmt.Errorf("%w: release_scheme %d", ErrInvalidField, v) + } + h.ReleaseScheme = SchemeFiles case fSignature: h.Signature = make([]byte, signatureSize) err = readFixed(f, h.Signature, "signature") @@ -240,6 +259,8 @@ func (h *Header) validate(maxLife time.Duration) error { return fmt.Errorf("%w: zero network id", ErrInvalidField) case h.ReleaseID == ReleaseID{}: return fmt.Errorf("%w: zero release id", ErrInvalidField) + case h.ReleaseScheme > SchemeFiles: + return fmt.Errorf("%w: release scheme", ErrInvalidField) case h.PublisherID.IsZero(): return fmt.Errorf("%w: zero publisher id", ErrInvalidField) case len(h.PublisherKey) != publicKeySize: @@ -299,8 +320,13 @@ func sign(key *identity.Key, tag string, nw netid.NetworkID, body []byte) ([]byt // ContentHash is the SHA-256 of an encoded record. func ContentHash(encoded []byte) [32]byte { return sha256.Sum256(encoded) } -// newHeader fills the publisher and time fields for a builder. -func newHeader(key *identity.Key, nw netid.NetworkID, rid ReleaseID, now time.Time, life time.Duration) (Header, error) { +// newHeader fills the publisher and time fields for a builder. The scheme is +// stored in canonical form: the default (nzb) is zero, which is what a decoder +// produces when the field is absent. +func newHeader(key *identity.Key, nw netid.NetworkID, rid ReleaseID, scheme uint8, now time.Time, life time.Duration) (Header, error) { + if scheme == SchemeNZB { + scheme = 0 + } if key == nil { return Header{}, fmt.Errorf("%w: nil key", ErrInvalidField) } @@ -312,5 +338,16 @@ func newHeader(key *identity.Key, nw netid.NetworkID, rid ReleaseID, now time.Ti PublisherKey: key.PublicKey(), CreatedAt: created, ExpiresAt: created.Add(life), + + ReleaseScheme: scheme, }, nil } + +// Scheme returns how the Release ID was derived: SchemeNZB when the field is +// absent, SchemeFiles when it is 2. +func (h *Header) Scheme() uint8 { + if h.ReleaseScheme == SchemeFiles { + return SchemeFiles + } + return SchemeNZB +} diff --git a/internal/peernet/recsync/bundle_access_test.go b/internal/peernet/recsync/bundle_access_test.go new file mode 100644 index 000000000..d4d7cf35e --- /dev/null +++ b/internal/peernet/recsync/bundle_access_test.go @@ -0,0 +1,31 @@ +package recsync + +import "testing" + +func TestExchangeNetworkLevelAndPeerBackedOff(t *testing.T) { + n := newTNode(t, nodeOpts{}) + nw := netOf(5) + if _, ok := n.x.NetworkLevel(nw); ok { + t.Fatal("unregistered network has a level") + } + n.join(nw, 3) + if lvl, ok := n.x.NetworkLevel(nw); !ok || lvl != 3 { + t.Fatalf("level = %d, %v", lvl, ok) + } + n.join(nw, 0) + if _, ok := n.x.NetworkLevel(nw); ok { + t.Fatal("level 0 network still registered") + } + + other := mustKey(t).NodeID() + if n.x.PeerBackedOff(nw, other, "") { + t.Fatal("fresh peer is backed off") + } + n.x.penalize(nw, other) + if !n.x.PeerBackedOff(nw, other, "") { + t.Fatal("penalised peer is not backed off") + } + if n.x.PeerBackedOff(netOf(6), other, "") { + t.Fatal("back-off leaked to another network") + } +} diff --git a/internal/peernet/recsync/exchange.go b/internal/peernet/recsync/exchange.go index 0785179db..b1c2649ac 100644 --- a/internal/peernet/recsync/exchange.go +++ b/internal/peernet/recsync/exchange.go @@ -177,6 +177,20 @@ func (x *Exchange) network(id netid.NetworkID) (NetworkInfo, bool) { return info, ok } +// NetworkLevel returns the exposure level of a registered network. A network +// that is not registered (unknown, off, or removed by its host) is not found. +func (x *Exchange) NetworkLevel(id netid.NetworkID) (int, bool) { + info, ok := x.network(id) + return info.Level, ok +} + +// PeerBackedOff reports whether peer, or the remote address ip (empty when +// unknown), is currently backed off for sending hostile records in nw. Other +// services use it to refuse a peer record exchange has already struck. +func (x *Exchange) PeerBackedOff(nw netid.NetworkID, peer identity.NodeID, ip string) bool { + return x.pusherBackedOff(nw, peer, ip) +} + // networks returns the registered networks sorted by ID. func (x *Exchange) networks() []NetworkInfo { x.mu.Lock() diff --git a/internal/peernet/recsync/ingest.go b/internal/peernet/recsync/ingest.go index 08fe98dfc..fe4bd397f 100644 --- a/internal/peernet/recsync/ingest.go +++ b/internal/peernet/recsync/ingest.go @@ -287,6 +287,7 @@ func (i *Ingester) verifiedRow(nw netid.NetworkID, raw []byte, env records.Envel NetworkID: nw.String(), ContentHash: hex.EncodeToString(env.ContentHash[:]), ReleaseID: hex.EncodeToString(env.ReleaseID[:]), PublisherNodeID: env.PublisherID.String(), CreatedAt: env.CreatedAt, ExpiresAt: env.ExpiresAt, Source: source, Payload: raw, + ReleaseScheme: int(env.ReleaseScheme), } switch env.Kind { case records.KindRelease: @@ -298,6 +299,9 @@ func (i *Ingester) verifiedRow(nw netid.NetworkID, raw []byte, env records.Envel return row, err } row.Kind = database.PeernetRecordKindRelease + if r.BundleID != (records.BundleID{}) { + row.BundleID = hex.EncodeToString(r.BundleID[:]) + } row.Title, row.TotalSize, row.IMDbID = r.Title, r.TotalSize, r.Media.IMDb row.Season, row.Episode = intPtr(r.Media.Season), intPtr(r.Media.Episode) case records.KindAttestation: diff --git a/internal/peernet/recsync/ingest_test.go b/internal/peernet/recsync/ingest_test.go index 254504430..31cfe4497 100644 --- a/internal/peernet/recsync/ingest_test.go +++ b/internal/peernet/recsync/ingest_test.go @@ -467,3 +467,30 @@ func TestIngestRateIsFairAcrossRelayers(t *testing.T) { t.Fatalf("an honest relayer ingested %d of 2 records after another relayer flooded", n) } } + +func TestIngestStoresTheAnnouncedBundleIDFromTheSignedRecord(t *testing.T) { + pub, self := mustKey(t), mustKey(t) + nw := netOf(1) + st := &fakeStore{} + ing := newTestIngester(t, self.NodeID(), st) + in := relInput(9) + in.BundleID = records.BundleID{1, 2, 3} + _, raw, err := records.BuildReleaseRecord(pub, nw, in, testNow) + if err != nil { + t.Fatal(err) + } + if _, err := ing.Ingest(context.Background(), nw, raw, OriginPeer); err != nil { + t.Fatal(err) + } + if got, want := st.rows[0].BundleID, hexOf(in.BundleID); got != want { + t.Fatalf("stored bundle id = %q, want %q", got, want) + } + // A record without one stores none. + raw = buildRelease(t, pub, nw, testNow) + if _, err := ing.Ingest(context.Background(), nw, raw, OriginPeer); err != nil { + t.Fatal(err) + } + if got := st.rows[1].BundleID; got != "" { + t.Fatalf("bundle id of a record without one = %q", got) + } +} diff --git a/internal/peernet/recsync/policy.go b/internal/peernet/recsync/policy.go index c6872ef1d..7148b1456 100644 --- a/internal/peernet/recsync/policy.go +++ b/internal/peernet/recsync/policy.go @@ -7,7 +7,7 @@ package recsync // 0 no no no no no no routes answer as if the network were unknown // 1 yes yes yes no no no receive only: serves an empty page // 2 yes yes yes yes yes yes own and relayed records served -// 3 yes yes yes yes yes yes same as 2 for records +// 3 yes yes yes yes yes yes same as 2 for records, and serves its bundles // // A level 1 node accepts pushes because "receives" does not name a channel // and an outbound-only publisher can reach a peer only by pushing; what it @@ -25,6 +25,8 @@ type Policy struct { Push bool // Serve: this node serves its record list (own and relayed) to pullers. Serve bool + // ServeBundles: this node serves finished-import bundles it announced. + ServeBundles bool } // PolicyFor returns the policy of an exposure level. Out-of-range levels fail @@ -33,8 +35,10 @@ func PolicyFor(level int) Policy { switch level { case 1: return Policy{Exists: true, Pull: true, AcceptPush: true} - case 2, 3: + case 2: return Policy{Exists: true, Pull: true, AcceptPush: true, Publish: true, Push: true, Serve: true} + case 3: + return Policy{Exists: true, Pull: true, AcceptPush: true, Publish: true, Push: true, Serve: true, ServeBundles: true} } return Policy{} } diff --git a/internal/peernet/recsync/policy_test.go b/internal/peernet/recsync/policy_test.go index 63aac4fb0..fd4d69341 100644 --- a/internal/peernet/recsync/policy_test.go +++ b/internal/peernet/recsync/policy_test.go @@ -15,8 +15,8 @@ func TestPolicyFor(t *testing.T) { // Level 2: shares metadata. Publishes its own records, pushes them, // and serves own and relayed records. {2, Policy{Exists: true, Pull: true, AcceptPush: true, Publish: true, Push: true, Serve: true}}, - // Level 3 behaves like 2 for records (bundles come later). - {3, Policy{Exists: true, Pull: true, AcceptPush: true, Publish: true, Push: true, Serve: true}}, + // Level 3 behaves like 2 for records and also serves bundles. + {3, Policy{Exists: true, Pull: true, AcceptPush: true, Publish: true, Push: true, Serve: true, ServeBundles: true}}, // Anything out of range fails closed. {-1, Policy{}}, {4, Policy{}}, @@ -31,7 +31,15 @@ func TestPolicyFor(t *testing.T) { func TestPolicyLevelOneNeverPublishesOrServes(t *testing.T) { p := PolicyFor(1) - if p.Publish || p.Push || p.Serve { + if p.Publish || p.Push || p.Serve || p.ServeBundles { t.Fatalf("level 1 must publish and serve nothing: %+v", p) } } + +func TestOnlyLevelThreeServesBundles(t *testing.T) { + for level := -1; level <= 4; level++ { + if got, want := PolicyFor(level).ServeBundles, level == 3; got != want { + t.Errorf("level %d: ServeBundles = %v, want %v", level, got, want) + } + } +} diff --git a/internal/peernet/service/bundle_import_e2e_test.go b/internal/peernet/service/bundle_import_e2e_test.go new file mode 100644 index 000000000..a76388783 --- /dev/null +++ b/internal/peernet/service/bundle_import_e2e_test.go @@ -0,0 +1,279 @@ +package service + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + + "github.com/javi11/nntppool/v5" + "github.com/kipsilabs/altmount/internal/config" + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/metadata" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/bundleimport" + "github.com/kipsilabs/altmount/internal/peernet/bundlex" + "github.com/kipsilabs/altmount/internal/peernet/contentid" + "github.com/kipsilabs/altmount/internal/pool" + "github.com/kipsilabs/altmount/internal/testsupport/fakepool" +) + +// e2ePool serves one fake NNTP client; a method the stack should not call +// panics (the embedded interface is nil). +type e2ePool struct { + pool.Manager + client pool.NntpClient +} + +func (m *e2ePool) GetPool() (pool.NntpClient, error) { return m.client, nil } +func (m *e2ePool) HasPool() bool { return true } +func (m *e2ePool) IncArticlesDownloaded() {} +func (m *e2ePool) UpdateDownloadProgress(_ string, _ int64) {} +func (m *e2ePool) SpeculativeBudget() *pool.SpeculativeBudget { return nil } +func (m *e2ePool) NotifyStreamChange() {} +func (m *e2ePool) SetStreamSource(_ pool.StreamActivitySource) {} +func (m *e2ePool) AcquireImportSlot(context.Context) (func(), error) { + return func() {}, nil +} +func (m *e2ePool) AcquireImportConnection(context.Context) (func(), error) { + return func() {}, nil +} +func (m *e2ePool) StatSweepConcurrency(c int) int { return c } + +type e2eProber struct{} + +func (e2eProber) ProbeRelease(context.Context, []bundleimport.ProbeFile) error { return nil } + +type e2eAnnouncer struct { + mu sync.Mutex + got []bundleimport.Announcement +} + +func (a *e2eAnnouncer) AnnounceVerifiedImport(_ context.Context, an bundleimport.Announcement) error { + a.mu.Lock() + defer a.mu.Unlock() + a.got = append(a.got, an) + return nil +} +func (a *e2eAnnouncer) count() int { a.mu.Lock(); defer a.mu.Unlock(); return len(a.got) } + +type e2eRecorder struct { + mu sync.Mutex + recs []bundleimport.ImportRecord +} + +func (r *e2eRecorder) RecordPeerImport(_ context.Context, rec bundleimport.ImportRecord) (int64, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.recs = append(r.recs, rec) + return int64(100 + len(r.recs)), nil +} + +// serviceFetcher is the bridge adapter (bridge imports this package, so the +// test cannot use it): FetchSource becomes the importer's Origin. +type serviceFetcher struct{ svc *Service } + +func (f serviceFetcher) FetchBundle(ctx context.Context, rel string) (*bundle.Bundle, bundleimport.Origin, error) { + b, src, err := f.svc.FetchBundle(ctx, rel) + if err != nil { + return nil, bundleimport.Origin{}, err + } + return b, bundleimport.Origin{NetworkID: src.Network.String(), PublisherID: src.Publisher.String()}, nil +} + +type serviceReporter struct{ svc *Service } + +func (r serviceReporter) ReportBadBundle(_ context.Context, o bundleimport.Origin, _ string) { + r.svc.NoteBundleFullCheck(o.NetworkID, o.PublisherID, false) +} +func (r serviceReporter) ReportVerifiedBundle(_ context.Context, o bundleimport.Origin, _ string) { + r.svc.NoteBundleFullCheck(o.NetworkID, o.PublisherID, true) +} + +type e2eReceiver struct { + *instance + imp *bundleimport.Importer + lib *metadata.MetadataService + ann *e2eAnnouncer + rec *e2eRecorder + idx *database.PeernetContentRepository + pool *fakepool.Client +} + +// mkvFile is a structurally valid Matroska file of size bytes: EBML header, a +// Segment ending at the file end, a SeekHead pointing at Info and at Cues (the +// last element), and a Cluster of filler between them. +func mkvFile(size int) []byte { + u32 := func(v int) []byte { return []byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)} } + seek := func(id []byte, pos int) []byte { + body := append(append([]byte{0x53, 0xAB, 0x84}, id...), append([]byte{0x53, 0xAC, 0x84}, u32(pos)...)...) + return append([]byte{0x4D, 0xBB, byte(0x80 | len(body))}, body...) + } + info := []byte{0x15, 0x49, 0xA9, 0x66} + cues := []byte{0x1C, 0x53, 0xBB, 0x6B} + ebml := append([]byte{0x1A, 0x45, 0xDF, 0xA3, 0x8B, 0x42, 0x82, 0x88}, "matroska"...) + segStart := len(ebml) + 12 + seekHead := append([]byte{0x11, 0x4D, 0x9B, 0x74, byte(0x80 | 34)}, append(seek(info, 39), seek(cues, size-segStart-9)...)...) + out := append([]byte{}, ebml...) + segSize := size - segStart + out = append(out, 0x18, 0x53, 0x80, 0x67, 0x01, 0, byte(segSize>>40), byte(segSize>>32), byte(segSize>>24), byte(segSize>>16), byte(segSize>>8), byte(segSize)) + out = append(out, seekHead...) + out = append(out, append(append([]byte{}, info...), 0x84, 0, 0, 0, 0)...) + // A Cluster whose 8-byte size covers the filler. + fill := size - len(out) - 12 - 9 + out = append(out, 0x1F, 0x43, 0xB6, 0x75, 0x01, 0, byte(fill>>40), byte(fill>>32), byte(fill>>24), byte(fill>>16), byte(fill>>8), byte(fill)) + out = append(out, make([]byte, fill)...) + return append(out, append(append([]byte{}, cues...), 0x84, 0, 0, 0, 0)...) +} + +// newE2EReceiver wires a level-1 node's bundle importer over real sqlite, a +// real metadata service in a temp dir and a fake NNTP pool that serves node +// A's articles. +func newE2EReceiver(t *testing.T, b *instance, serve bool) *e2eReceiver { + t.Helper() + fp := fakepool.New() + if serve { + file := mkvFile(12 * bundleSegSize) + for i := 0; i < 12; i++ { + fp.SetBehavior(strings.Replace("seg-NNN@news.test", "NNN", pad3(i), 1), fakepool.SegmentBehavior{ + Bytes: file[i*bundleSegSize : (i+1)*bundleSegSize], + YEnc: nntppool.YEncMeta{PartBegin: int64(i * bundleSegSize), PartSize: bundleSegSize, FileSize: int64(len(file))}}) + } + } + lib := metadata.NewMetadataService(t.TempDir()) + conn, d := b.db.Connection(), b.db.Dialect() + r := &e2eReceiver{instance: b, lib: lib, ann: &e2eAnnouncer{}, rec: &e2eRecorder{}, idx: database.NewPeernetContentRepository(conn, d), pool: fp} + cfg := config.DefaultConfig() + r.imp = bundleimport.New(bundleimport.Config{ + Library: lib, + Headers: bundleimport.NewPoolHeaderReader(&e2ePool{client: fp}), + Stager: bundleimport.NewFSStager(bundleimport.FSStagerDeps{Pool: &e2ePool{client: fp}, Config: func() *config.Config { return cfg }, Root: t.TempDir()}), + Prober: e2eProber{}, State: database.NewPeernetBundleImportRepository(conn, d), Index: r.idx, + Fetcher: serviceFetcher{b.svc}, Recorder: r.rec, Announcer: r.ann, Reporter: serviceReporter{b.svc}, + HealthEnabled: func() bool { return true }, + }) + return r +} + +func pad3(i int) string { + return string([]byte{byte('0' + i/100), byte('0' + i/10%10), byte('0' + i%10)}) +} + +func TestBundleImportEndToEnd_FullCheckPassesAnnouncesOnceAndBecomesPublishEligible(t *testing.T) { + ctx := context.Background() + a := newBundlePublisher(t, 3, nil) + a.publishHealthy(t) + b := bundleReceiver(t, a) + b.syncRound(t) + r := newE2EReceiver(t, b, true) + + res, err := r.imp.ImportRelease(ctx, a.rel, bundleimport.Target{Category: "movies"}) + if err != nil { + t.Fatalf("import: %v", err) + } + if len(res.Paths) != 1 || !strings.HasPrefix(res.Paths[0], "movies/") { + t.Fatalf("paths %v", res.Paths) + } + path := res.Paths[0] + // In B's library, playable, with a verified release identity in the index. + m, err := r.lib.ReadFileMetadata(path) + if err != nil || m == nil || m.FileSize != 12*bundleSegSize { + t.Fatalf("file not in the library: %v %+v", err, m) + } + row, ok, err := r.idx.GetByPath(ctx, path) + if err != nil || !ok || row.ReleaseID != a.rel || !contentid.IsVerifiedBundleRow(row) { + t.Fatalf("index row %+v ok=%v err=%v", row, ok, err) + } + // Peer-origin pending: not published, not announced, held at the ARRs. + if from, passed, _ := r.imp.PeerBundleStatus(ctx, path); !from || passed { + t.Fatalf("pending state wrong: from=%v passed=%v", from, passed) + } + if r.ann.count() != 0 || !r.imp.QueueHasPending(ctx, 101) { + t.Fatal("the ARRs must not be told before the full check") + } + if len(r.rec.recs) != 1 || r.rec.recs[0].ReleaseID != a.rel || r.rec.recs[0].Title == "" { + t.Fatalf("recorded import %+v", r.rec.recs) + } + + // The health worker's forced full check passes. + r.imp.FullCheckPassed(ctx, path) + r.imp.FullCheckPassed(ctx, path) // idempotent + if r.ann.count() != 1 { + t.Fatalf("announced %d times, want exactly once", r.ann.count()) + } + if from, passed, err := r.imp.PeerBundleStatus(ctx, path); err != nil || !from || !passed { + t.Fatalf("not publish-eligible after the pass: %v %v %v", from, passed, err) + } + if c := r.imp.Counters(); c.Accepted != 1 || c.FullCheckPassed != 1 || c.Announced != 1 || c.BadBundleReports != 0 { + t.Fatalf("counters %+v", c) + } + // A later verified bundle is not penalised: A is still fetchable. + if _, _, err := b.svc.FetchBundle(ctx, a.rel); err != nil { + t.Fatalf("publisher wrongly penalised: %v", err) + } +} + +func TestBundleImportEndToEnd_FullCheckFailureBacksThePublisherOffForFetches(t *testing.T) { + ctx := context.Background() + a := newBundlePublisher(t, 3, nil) + a.publishHealthy(t) + b := bundleReceiver(t, a) + b.syncRound(t) + r := newE2EReceiver(t, b, true) + + res, err := r.imp.ImportRelease(ctx, a.rel, bundleimport.Target{Category: "movies"}) + if err != nil { + t.Fatal(err) + } + path := res.Paths[0] + r.imp.FullCheckFailed(ctx, path) + r.imp.FullCheckFailed(ctx, path) // reported once per release + + if r.ann.count() != 0 { + t.Fatal("a failed import is never announced") + } + // The failed import is removed with its state: nothing of it can be published. + if _, passed, _ := r.imp.PeerBundleStatus(ctx, path); passed { + t.Fatal("a failed file must never become publish-eligible") + } + if r.lib.FileExists(path) { + t.Fatal("a failed import must not stay in the library") + } + if _, _, err := b.svc.FetchBundle(ctx, a.rel); !errors.Is(err, bundlex.ErrNoOffers) { + t.Fatalf("the publisher must be backed off for fetches: %v", err) + } + st := b.statusOf(t).Bundles + if st.FullCheckReports != 1 || st.FullCheckBackedOff != 1 { + t.Fatalf("status %+v", st) + } + if c := r.imp.Counters(); c.FullCheckFailed != 1 || c.BadBundleReports != 1 { + t.Fatalf("counters %+v", c) + } + // Record sync is unaffected: the offer is still known. + if offers, err := b.svc.BundleOffers(ctx, a.rel); err != nil || len(offers) != 1 { + t.Fatalf("offers %+v %v", offers, err) + } +} + +func TestBundleImportEndToEnd_ArticlesMissingOnTheReceiversProvidersRejectsAndLeavesNothing(t *testing.T) { + ctx := context.Background() + a := newBundlePublisher(t, 3, nil) + a.publishHealthy(t) + b := bundleReceiver(t, a) + b.syncRound(t) + r := newE2EReceiver(t, b, false) // B's providers hold none of A's articles + + if _, err := r.imp.ImportRelease(ctx, a.rel, bundleimport.Target{Category: "movies"}); err == nil { + t.Fatal("import must be rejected: the media head is not on B's providers") + } + if got, err := r.lib.ReadFileMetadata("movies/Some.Movie.2020.1080p/movie.mkv"); err == nil && got != nil { + t.Fatal("a rejected import left a file behind") + } + if _, ok, _ := r.idx.GetByPath(ctx, "movies/Some.Movie.2020.1080p/movie.mkv"); ok { + t.Fatal("a rejected import left an index row") + } + if len(r.rec.recs) != 0 || r.ann.count() != 0 { + t.Fatal("a rejected import must not be recorded or announced") + } +} diff --git a/internal/peernet/service/bundles.go b/internal/peernet/service/bundles.go new file mode 100644 index 000000000..908873e9d --- /dev/null +++ b/internal/peernet/service/bundles.go @@ -0,0 +1,324 @@ +package service + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/bundlex" + "github.com/kipsilabs/altmount/internal/peernet/contentid" + "github.com/kipsilabs/altmount/internal/peernet/identity" + "github.com/kipsilabs/altmount/internal/peernet/netid" + "github.com/kipsilabs/altmount/internal/peernet/publish" + "github.com/kipsilabs/altmount/internal/peernet/records" + "github.com/kipsilabs/altmount/internal/peernet/recsync" +) + +// FetchSource says which network and publisher a fetched bundle came from, so +// the import can be marked as coming from the peer network. +type FetchSource = bundlex.FetchSource + +// Offer is a stored release record that announces a bundle for a release. +type Offer = bundlex.Offer + +// Errors of FetchBundle. +var ( + ErrBundlesUnavailable = errors.New("peer network bundles are not configured") + ErrNoBundleOffers = bundlex.ErrNoOffers + ErrFetchRateLimited = bundlex.ErrRateLimited + ErrFetchBusy = bundlex.ErrFetchBusy + ErrBundleFetchFailed = bundlex.ErrAllFailed +) + +// knownPeerAge is how long a peer must have been verified to count as known. +const knownPeerAge = 24 * time.Hour + +// buildSourceProvider is implemented by the metadata host the production +// wiring passes. Its method is unexported on purpose: only this package can +// obtain the full-metadata reader, and only to hand it to the bundle builder. +type buildSourceProvider interface { + bundleBuildSource() bundlex.BuildSource +} + +// BundleStatus counts bundle activity since start. It holds no bundle content, +// path or name. +type BundleStatus struct { + // Built counts completed builds; Cached and CachedBytes the bundles held in + // memory now; CacheHits the builds avoided. + Built, CacheHits, BuildFailures int64 + Cached, CachedBytes int + // Announced is how many release records announce a bundle. + Announced int + // NotShareable counts releases that cannot be shared as a bundle, by reason. + NotShareable map[string]int64 + // Served and ServedBytes count bundles sent; ServeRefused counts refused + // requests by reason. + Served, ServedBytes int64 + ServeRefused map[string]int64 + // Fetched counts verified downloads; FetchFailed failures by reason; + // FetchRateLimited calls refused by the fetch limits; FetchBackedOff the + // publishers currently skipped. + Fetched, FetchRateLimited int64 + FetchFailed map[string]int64 + FetchBackedOff int + // FullCheckReports counts verified bundles whose release then failed the + // local full check; FullCheckBackedOff the publishers skipped for fetches + // because of that now. + FullCheckReports int64 + FullCheckBackedOff int + // Import counts what the receiving side did with fetched bundles (the + // counters of the bundle importer); zero while it is not wired. + Import BundleImportStatus +} + +// BundleImportStatus mirrors the counters of the bundle importer: bundles +// accepted as local imports, rejected (by reason), and how their full checks +// ended. It holds no bundle content, path or name. +type BundleImportStatus struct { + Accepted, AlreadyImported int64 + Rejected map[string]int64 + FullCheckPassed, FullCheckFailed int64 + Announced, BadBundleReports int64 + // Pending files still owe their full check; Stale are pending past the + // maximum age (they are failed, never announced). + Pending, Stale int64 +} + +type contentLibrary struct { + repo *database.PeernetContentRepository +} + +// ReleaseFiles implements bundlex.Library: one path per distinct File ID. +func (l contentLibrary) ReleaseFiles(ctx context.Context, releaseHex string) (int, []string, error) { + entries, err := l.repo.FindByReleaseID(ctx, releaseHex) + if err != nil || len(entries) == 0 { + return 0, nil, err + } + seen := make(map[string]struct{}, len(entries)) + paths := make([]string, 0, len(entries)) + for _, e := range entries { + if _, dup := seen[e.FileID]; dup { + continue + } + seen[e.FileID] = struct{}{} + paths = append(paths, e.FilePath) + } + return entries[0].ReleaseScheme, paths, nil +} + +// bundleSource adapts the builder to publish.BundleSource. +type bundleSource struct{ b *bundlex.Builder } + +func (s bundleSource) BundleFor(ctx context.Context, rel records.ReleaseID, info publish.ReleaseInfo) publish.BundleDecision { + built, err := s.b.Build(ctx, contentid.ID(rel).String(), info.Title, info.PostDate) + var ns *bundlex.NotShareableError + switch { + case errors.As(err, &ns): + return publish.BundleDecision{Skip: ns.Reason} + case err != nil: + return publish.BundleDecision{Err: err} + } + return publish.BundleDecision{ID: records.BundleID(built.ID)} +} + +// announcements adapts the publish state and record stores to +// bundlex.Announcements. +type announcements struct { + state *database.PeernetPublishStateRepository + recs *database.PeernetRecordRepository +} + +func (a announcements) Announced(ctx context.Context, nw netid.NetworkID, id bundle.BundleID) (bundlex.Announcement, bool, error) { + st, ok, err := a.state.GetReleaseByBundle(ctx, nw.String(), id.String()) + if err != nil || !ok { + return bundlex.Announcement{}, false, err + } + return bundlex.Announcement{ReleaseID: st.ReleaseID, OriginCategory: st.OriginCategory, OriginIndexer: st.OriginIndexer}, true, nil +} + +func (a announcements) Withdraw(ctx context.Context, nw netid.NetworkID, rel string) error { + if _, err := a.recs.DeleteOwnRelease(ctx, nw.String(), rel); err != nil { + return err + } + return a.state.DeleteRelease(ctx, nw.String(), rel) +} + +func (a announcements) MarkStale(ctx context.Context, nw netid.NetworkID, rel string) error { + return a.state.MarkBundleStale(ctx, nw.String(), rel) +} + +// bundleProvider builds the bundle of a release from the same release +// information that went into its record. +type bundleProvider struct { + b *bundlex.Builder + info *infoSource +} + +func (p bundleProvider) Bundle(ctx context.Context, releaseHex string) (*bundlex.Built, error) { + info, err := p.info.forRelease(ctx, releaseHex, "") + if err != nil { + return nil, fmt.Errorf("release information: %w", err) + } + return p.b.BuildForServe(ctx, releaseHex, info.Title, info.PostDate) +} + +// peerBook adapts the peer repository to bundlex.PeerBook. +type peerBook struct { + store *database.PeernetPeerRepository + maxFailures int +} + +func (p peerBook) VerifiedPeer(ctx context.Context, nw netid.NetworkID, node identity.NodeID) (recsync.Peer, bool, error) { + row, err := p.store.Get(ctx, nw.String(), node.String()) + if err != nil || row == nil || !row.Verified || row.Address == "" || row.Failures >= p.maxFailures { + return recsync.Peer{}, false, err + } + trusted := row.Source == database.PeernetSourceUser || row.Source == database.PeernetSourceLocal + return recsync.Peer{NodeID: node, Address: row.Address, Trusted: trusted}, true, nil +} + +func (p peerBook) Known(ctx context.Context, nw netid.NetworkID, nodes []string) (map[string]bool, error) { + return p.store.KnownAmong(ctx, nw.String(), nodes, knownPeerAge, time.Now()) +} + +// startBuilder creates the bundle builder and the fetch client. Without a +// full-metadata source the builder is absent: level 3 then announces no +// bundles. +func (n *node) startBuilder() { + d := n.deps + if n.rx == nil || d.Content == nil { + return + } + n.fetcher = bundlex.NewFetcher(bundlex.FetchConfig{ + Self: n.nodeID, Store: d.Records, Peers: peerBook{store: d.Peers, maxFailures: recsync.DefaultLimits().MaxPeerFailures}, + Networks: n.rx, Dialer: n.client, Auth: authBridge{mem: n.mem, onRefused: n.noteRefused}, + }) + src := d.BundleSource + if src == nil { + if p, ok := d.Metadata.(buildSourceProvider); ok { + src = p.bundleBuildSource() + } + } + if src == nil { + return + } + n.builder = bundlex.NewBuilder(bundlex.BuilderConfig{Source: src, Library: contentLibrary{repo: d.Content}, Patches: d.Patches}) +} + +// startBundleServe creates the serve route handler. It needs the publishing +// pipeline (release information) and the publish state (what was announced). +func (n *node) startBundleServe() { + d := n.deps + if n.builder == nil || n.pipe == nil || n.rx == nil || d.PublishState == nil || d.Records == nil { + return + } + var prov bundlex.Provider = bundleProvider{b: n.builder, info: n.pipe.info} + if d.wrapProvider != nil { + prov = d.wrapProvider(prov) + } + n.bsrv = bundlex.NewServer(bundlex.ServerConfig{ + Networks: n.rx, Auth: authBridge{mem: n.mem, onRefused: n.noteRefused}, + Announcements: announcements{state: d.PublishState, recs: d.Records}, + Provider: prov, + Exclusions: d.Exclusions, BackedOff: n.rx.PeerBackedOff, Limits: d.bundleServeLimits, + Log: n.logChange, + }) +} + +// invalidateBundles drops cached bundles of releases whose identity changed. +func (s *Service) invalidateBundles(releaseIDs []string) { + n := s.currentNode() + if n == nil || n.builder == nil { + return + } + for _, id := range releaseIDs { + n.builder.Invalidate(id) + } +} + +// BundleOffers lists the stored release records that announce a bundle for +// releaseID, best first (known publishers, then those with a fresh healthy +// attestation, then newest). Offer.Fetchable says whether the publisher is a +// verified peer this node can dial; a record relayed from a publisher with no +// verified address is listed but never fetched. +func (s *Service) BundleOffers(ctx context.Context, releaseID string) ([]Offer, error) { + n, err := s.requireNode() + if err != nil { + return nil, err + } + if n.fetcher == nil { + return nil, ErrBundlesUnavailable + } + return n.fetcher.Offers(ctx, releaseID) +} + +// FetchBundle downloads the bundle of releaseID on demand from the best +// offers (at most three attempts), verifies it against the Bundle ID and +// Release ID of the signed record, and returns it with its source. It is rate +// limited and concurrency limited, and is never called speculatively by the +// service itself. +func (s *Service) FetchBundle(ctx context.Context, releaseID string) (*bundle.Bundle, FetchSource, error) { + n, err := s.requireNode() + if err != nil { + return nil, FetchSource{}, err + } + if n.fetcher == nil { + return nil, FetchSource{}, ErrBundlesUnavailable + } + return n.fetcher.Fetch(ctx, releaseID) +} + +// NoteBundleFullCheck tells the fetcher how the full local check of a release +// imported from publisherID's bundle in networkID ended. A failure backs that +// publisher off for bundle fetches only (never record sync, never a purge); a +// pass clears an earlier penalty. Unparseable identifiers are ignored. +func (s *Service) NoteBundleFullCheck(networkID, publisherID string, passed bool) { + n := s.currentNode() + if n == nil || n.fetcher == nil { + return + } + nw, err := netid.ParseNetworkID(networkID) + if err != nil { + return + } + pub, err := identity.ParseNodeID(publisherID) + if err != nil { + return + } + if passed { + n.fetcher.ReportVerifiedBundle(nw, pub) + return + } + n.fetcher.ReportBadBundle(nw, pub) +} + +// bundleStatus gathers the bundle counters. +func (n *node) bundleStatus(ctx context.Context) BundleStatus { + var st BundleStatus + if n.builder != nil { + b := n.builder.Status() + st.Built, st.CacheHits, st.BuildFailures = b.Built, b.CacheHits, b.BuildFailures + st.Cached, st.CachedBytes, st.NotShareable = b.Cached, b.CachedBytes, b.NotShareable + } + if n.bsrv != nil { + s := n.bsrv.Status() + st.Served, st.ServedBytes, st.ServeRefused = s.Served, s.Bytes, s.Refused + } + if n.fetcher != nil { + f := n.fetcher.Status() + st.Fetched, st.FetchFailed, st.FetchRateLimited, st.FetchBackedOff = f.OK, f.Failed, f.RateLimited, f.BackedOff + st.FullCheckReports, st.FullCheckBackedOff = f.FullCheckReports, f.FullCheckBackedOff + } + if n.deps.BundleImportStatus != nil { + st.Import = n.deps.BundleImportStatus() + } + if n.deps.PublishState != nil { + if c, err := n.deps.PublishState.CountAnnouncedBundles(ctx); err == nil { + st.Announced = c + } + } + return st +} diff --git a/internal/peernet/service/bundles_test.go b/internal/peernet/service/bundles_test.go new file mode 100644 index 000000000..4e5ae245e --- /dev/null +++ b/internal/peernet/service/bundles_test.go @@ -0,0 +1,395 @@ +package service + +import ( + "bytes" + "context" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/database" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/bundlex" + "github.com/kipsilabs/altmount/internal/peernet/contentid" + "github.com/kipsilabs/altmount/internal/peernet/netid" + "github.com/kipsilabs/altmount/internal/peernet/publish" + "github.com/kipsilabs/altmount/internal/peernet/records" +) + +const bundleSegSize = 716800 + +var bundleMarkers = []string{"/local/store/path", "RCLONE-PW-MARK", "RCLONE-SALT-MARK", "dl-id-MARK", "IndexerMARK", + "CategoryMARK", "fingerprint-MARK", "health-MARK"} + +type allHealthy struct{} + +func (allHealthy) Statuses(_ context.Context, paths []string) (map[string]string, error) { + out := map[string]string{} + for _, p := range paths { + out[p] = string(database.HealthStatusHealthy) + } + return out, nil +} + +// bundleSrc is the fake full-metadata reader of the builder. +type bundleSrc struct { + meta map[string]*metapb.FileMetadata + nzb [][]string +} + +func (s *bundleSrc) ReadFile(p string) (*metapb.FileMetadata, error) { return s.meta[p], nil } +func (s *bundleSrc) NZBSegmentIDs(string) ([][]string, error) { return s.nzb, nil } +func (s *bundleSrc) NZBPassword(string, *metapb.FileMetadata) (string, bool) { + return "", false +} + +func bundleMeta(segs []string) *metapb.FileMetadata { + m := &metapb.FileMetadata{FileSize: int64(len(segs)) * bundleSegSize, SourceNzbPath: "/local/store/path/x.nzbz", + StoreRef: "/local/store/path/ref.nzbz", Status: metapb.FileStatus_FILE_STATUS_HEALTHY, ReleaseDate: 1793491200, + OriginalFilename: "movie.mkv", NzbdavId: "dl-id-MARK", Password: "RCLONE-PW-MARK", Salt: "RCLONE-SALT-MARK", + HoleProviderFingerprint: "fingerprint-MARK"} + for i := range segs { + m.SegmentData = append(m.SegmentData, &metapb.SegmentData{Id: segs[i], SegmentSize: bundleSegSize, EndOffset: bundleSegSize - 1}) + } + return m +} + +type bundlePub struct { + *pubFixture + src *bundleSrc + rel string +} + +// newBundlePublisher is a listening node at the given open-network level whose +// library holds one releasable file with real IDs. +func newBundlePublisher(t *testing.T, lvl int, mut func(*Deps)) *bundlePub { + t.Helper() + segs := make([]string, 12) + for i := range segs { + segs[i] = fmt.Sprintf("seg-%03d@news.test", i) + } + par := []string{"par-000@news.test"} + meta := bundleMeta(segs) + fid, err := contentid.FileID(meta) + if err != nil { + t.Fatal(err) + } + nzb := [][]string{segs, par} + store := &metapb.NzbStore{} + for _, f := range nzb { + fe := &metapb.NzbFileEntry{} + for _, id := range f { + fe.Segments = append(fe.Segments, &metapb.NzbSeg{Id: id}) + } + store.Files = append(store.Files, fe) + } + rid, err := contentid.ReleaseID(store) + if err != nil { + t.Fatal(err) + } + src := &bundleSrc{meta: map[string]*metapb.FileMetadata{pubPath: meta}, nzb: nzb} + set := Settings{Enabled: true, AcceptIncoming: true, OpenLevel: lvl} + f := newPubInstance(t, set, func(d *Deps) { + d.Health, d.BundleSource, d.Patches = allHealthy{}, src, bundlex.NoPatches{} + if mut != nil { + mut(d) + } + }) + f.seedFile(t, func(s *seed) { + s.release, s.file, s.category, s.indexer, s.downloadID = rid.String(), fid.String(), "CategoryMARK", "IndexerMARK", "dl-id-MARK" + s.size = meta.FileSize + }) + f.start(t) + if n := f.node(t); lvl > 0 && (n.builder == nil || n.bsrv == nil || n.pipe == nil) { + t.Fatalf("bundle wiring missing: builder=%v server=%v pipeline=%v", n.builder != nil, n.bsrv != nil, n.pipe != nil) + } + set.AdvertiseAddress, set.ListenPort = f.addr(t), f.port(t) + f.svc.Apply(context.Background(), set) + return &bundlePub{pubFixture: f, src: src, rel: rid.String()} +} + +func (p *bundlePub) publishHealthy(t *testing.T) { + t.Helper() + p.svc.NotifyHealthy(healthyNotice()) + p.drain(t) +} + +// bundleReceiver is a level-1 node connected to a. +func bundleReceiver(t *testing.T, a *bundlePub) *instance { + t.Helper() + b, bset := listeningInstance(t, Settings{OpenLevel: 1}) + bset.OpenPeers = []string{a.addr(t)} + b.svc.Apply(context.Background(), bset) + connect(t, b, a.instance, netid.Open()) + return b +} + +func releaseBundleIDs(t *testing.T, i *instance, local bool) []records.BundleID { + t.Helper() + var out []records.BundleID + for _, row := range i.stored(t, netid.Open()) { + env, err := records.Peek(row.Payload) + if err != nil || env.Kind != records.KindRelease { + continue + } + r, err := records.DecodeReleaseRecord(row.Payload) + if err != nil { + t.Fatal(err) + } + out = append(out, r.BundleID) + } + return out +} + +func TestBundleEndToEndLevelThreePublishesAndPeerFetches(t *testing.T) { + a := newBundlePublisher(t, 3, nil) + a.publishHealthy(t) + if a.status(t).Bundles.Announced != 1 { + t.Fatalf("bundles: %+v", a.status(t).Bundles) + } + b := bundleReceiver(t, a) + if res := b.syncRound(t); res.Stored != 2 { + t.Fatalf("sync: %+v", res) + } + offers, err := b.svc.BundleOffers(context.Background(), a.rel) + if err != nil || len(offers) != 1 || !offers[0].Fetchable { + t.Fatalf("offers %+v %v", offers, err) + } + got, src, err := b.svc.FetchBundle(context.Background(), a.rel) + if err != nil { + t.Fatal(err) + } + if src.Network != netid.Open() || src.Publisher.String() != a.id(t).String() { + t.Fatalf("source %+v", src) + } + built, err := a.node(t).builder.Build(context.Background(), a.rel, "Some.Movie.2020.1080p", time.Unix(1793491200, 0)) + if err != nil { + t.Fatal(err) + } + rid, _ := contentid.ParseID(a.rel) + want, err := bundle.Verify(built.Wire, built.ID, rid) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatal("the fetched bundle differs from the one the publisher built") + } + if s := a.status(t).Bundles; s.Served != 1 || s.ServedBytes != int64(len(built.Wire)) || s.Built != 1 { + t.Fatalf("publisher bundle status %+v", s) + } + if s := b.statusOf(t).Bundles; s.Fetched != 1 { + t.Fatalf("receiver bundle status %+v", s) + } +} + +func (i *instance) statusOf(t *testing.T) Status { + t.Helper() + st, err := i.svc.Status(context.Background()) + if err != nil { + t.Fatal(err) + } + return st +} + +func TestBundleLevelTwoPublisherAnnouncesNoBundleAndRefuses(t *testing.T) { + a := newBundlePublisher(t, 2, nil) + a.publishHealthy(t) + b := bundleReceiver(t, a) + b.syncRound(t) + for _, id := range releaseBundleIDs(t, b, false) { + if id != (records.BundleID{}) { + t.Fatal("a level 2 record carries a bundle id") + } + } + if _, _, err := b.svc.FetchBundle(context.Background(), a.rel); !errors.Is(err, bundlex.ErrNoOffers) { + t.Fatalf("fetch with no offer: %v", err) + } + if a.node(t).builder.Status().Built != 0 { + t.Fatal("a bundle was built at level 2") + } +} + +func TestBundleAnnouncedThenLevelDroppedIsRefusedImmediately(t *testing.T) { + a := newBundlePublisher(t, 3, nil) + a.publishHealthy(t) + b := bundleReceiver(t, a) + b.syncRound(t) + set := a.svc.currentSettings() + set.OpenLevel = 2 + a.svc.Apply(context.Background(), set) + _, _, err := b.svc.FetchBundle(context.Background(), a.rel) + if !errors.Is(err, bundlex.ErrAllFailed) { + t.Fatalf("fetch after the publisher dropped to level 2: %v", err) + } + if got := a.status(t).Bundles.ServeRefused["level"]; got != 1 { + t.Fatalf("refusals %+v", a.status(t).Bundles.ServeRefused) + } +} + +func TestBundleForgedBytesAreRejectedAndPublisherIsStruckWithoutHurtingRecordSync(t *testing.T) { + a := newBundlePublisher(t, 3, func(d *Deps) { + d.wrapProvider = func(p bundlex.Provider) bundlex.Provider { return forger{inner: p} } + }) + a.publishHealthy(t) + b := bundleReceiver(t, a) + b.syncRound(t) + if _, _, err := b.svc.FetchBundle(context.Background(), a.rel); !errors.Is(err, bundlex.ErrAllFailed) { + t.Fatalf("forged bundle accepted: %v", err) + } + st := b.statusOf(t).Bundles + if st.FetchFailed["verification_failed"] != 1 || st.FetchBackedOff != 1 || st.Fetched != 0 { + t.Fatalf("status %+v", st) + } + if _, _, err := b.svc.FetchBundle(context.Background(), a.rel); !errors.Is(err, bundlex.ErrNoOffers) { + t.Fatalf("struck publisher asked again: %v", err) + } + // Record sync with the same peer is unaffected by the bundle strike. + a.publish(t, netid.Open(), 77) + if res := b.syncRound(t); res.Stored != 1 { + t.Fatalf("record sync after a bundle strike: %+v", res) + } +} + +type forger struct{ inner bundlex.Provider } + +func (f forger) Bundle(ctx context.Context, rel string) (*bundlex.Built, error) { + real, err := f.inner.Bundle(ctx, rel) + if err != nil { + return nil, err + } + bad := *real + bad.Wire = bytes.Repeat([]byte{0x99}, len(real.Wire)) + return &bad, nil +} + +func TestBundleExclusionAddedAfterAnnouncementIsRefusedAndWithdrawn(t *testing.T) { + a := newBundlePublisher(t, 3, nil) + a.publishHealthy(t) + b := bundleReceiver(t, a) + b.syncRound(t) + before := a.node(t).builder.Status() + a.excl.set(publish.Exclusions{Indexers: []string{"indexermark"}}) + if _, _, err := b.svc.FetchBundle(context.Background(), a.rel); !errors.Is(err, bundlex.ErrAllFailed) { + t.Fatalf("excluded release served: %v", err) + } + if after := a.node(t).builder.Status(); after.Built != before.Built || after.CacheHits != before.CacheHits { + t.Fatalf("the excluded release was looked up again: %+v -> %+v", before, after) + } + if a.status(t).Bundles.Served != 0 { + t.Fatal("bytes were sent") + } + if n := a.openNetStatus(t).Records.Releases.Local; n != 0 { + t.Fatalf("record not withdrawn: %d", n) + } +} + +func TestBundleExcludedReleaseBuildsAndAnnouncesNothing(t *testing.T) { + a := newBundlePublisher(t, 3, nil) + a.excl.set(publish.Exclusions{Categories: []string{"CategoryMARK"}}) + a.publishHealthy(t) + if a.node(t).builder.Status().Built != 0 || a.status(t).Bundles.Announced != 0 { + t.Fatalf("%+v", a.status(t).Bundles) + } +} + +func TestBundleBytesServedContainNoLocalData(t *testing.T) { + a := newBundlePublisher(t, 3, nil) + a.publishHealthy(t) + built, err := a.node(t).builder.Build(context.Background(), a.rel, "Some.Movie.2020.1080p", time.Unix(1793491200, 0)) + if err != nil { + t.Fatal(err) + } + raw, err := bundle.Decompress(built.Wire) + if err != nil { + t.Fatal(err) + } + for _, m := range bundleMarkers { + if bytes.Contains(raw, []byte(m)) || bytes.Contains(built.Wire, []byte(m)) { + t.Errorf("bundle contains %q", m) + } + } + if strings.Contains(string(raw), pubPath) { + t.Error("bundle contains a virtual path") + } +} + +func TestBundleNotShareableReasonIsCountedAndRecordStillPublished(t *testing.T) { + a := newBundlePublisher(t, 3, func(d *Deps) { d.Patches = nil }) + a.publishHealthy(t) + st := a.status(t) + if st.Bundles.NotShareable["patch_status_unknown"] != 1 || st.Bundles.Announced != 0 { + t.Fatalf("%+v", st.Bundles) + } + if a.openNetStatus(t).Records.Releases.Local != 1 { + t.Fatal("the release record must still be published") + } +} + +func TestBundleLevelChangesSupersedeTheRecord(t *testing.T) { + ctx := context.Background() + a := newBundlePublisher(t, 2, nil) + a.publishHealthy(t) + awaitFirstUpkeep(t, a.pubFixture) + bundleOf := func() records.BundleID { + ids := releaseBundleIDs(t, a.instance, true) + if len(ids) != 1 { + t.Fatalf("%d release records", len(ids)) + } + return ids[0] + } + if bundleOf() != (records.BundleID{}) { + t.Fatal("level 2 record has a bundle id") + } + set := a.svc.currentSettings() + set.OpenLevel = 3 + a.svc.Apply(ctx, set) + time.Sleep(1100 * time.Millisecond) // records are stamped with whole seconds + a.node(t).maintainPublishing(ctx, false) + if bundleOf() == (records.BundleID{}) { + t.Fatal("2 -> 3 did not supersede the record with one carrying the bundle id") + } + set.OpenLevel = 2 + a.svc.Apply(ctx, set) + time.Sleep(1100 * time.Millisecond) + a.node(t).maintainPublishing(ctx, false) + if bundleOf() != (records.BundleID{}) { + t.Fatal("3 -> 2 left the bundle id announced") + } +} + +func TestBundleFullCheckFailureBacksPublisherOffForFetchesOnly(t *testing.T) { + a := newBundlePublisher(t, 3, nil) + a.publishHealthy(t) + b := bundleReceiver(t, a) + b.syncRound(t) + if _, _, err := b.svc.FetchBundle(context.Background(), a.rel); err != nil { + t.Fatal(err) + } + before := len(b.stored(t, netid.Open())) + b.svc.NoteBundleFullCheck(netid.Open().String(), a.id(t).String(), false) + if _, _, err := b.svc.FetchBundle(context.Background(), a.rel); !errors.Is(err, bundlex.ErrNoOffers) { + t.Fatalf("a publisher whose bundle failed the full check was asked again: %v", err) + } + if s := b.statusOf(t).Bundles; s.FullCheckReports != 1 || s.FullCheckBackedOff != 1 { + t.Fatalf("status %+v", s) + } + // Record sync and stored records are untouched. + b.syncRound(t) + if len(b.stored(t, netid.Open())) != before { + t.Fatal("the penalty changed the stored records") + } + if offers, err := b.svc.BundleOffers(context.Background(), a.rel); err != nil || len(offers) != 1 { + t.Fatalf("offers must stay listed: %+v %v", offers, err) + } + // A later verified bundle from the same publisher clears it. + b.svc.NoteBundleFullCheck(netid.Open().String(), a.id(t).String(), true) + if _, _, err := b.svc.FetchBundle(context.Background(), a.rel); err != nil { + t.Fatalf("not cleared: %v", err) + } + // Garbage identifiers are ignored. + b.svc.NoteBundleFullCheck("zz", "yy", false) +} diff --git a/internal/peernet/service/derive_index_test.go b/internal/peernet/service/derive_index_test.go new file mode 100644 index 000000000..ef08e9837 --- /dev/null +++ b/internal/peernet/service/derive_index_test.go @@ -0,0 +1,128 @@ +package service + +import ( + "context" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/metadata" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" +) + +func hx(c string) string { return strings.Repeat(c, 64) } + +func testDB(t *testing.T) *database.DB { + t.Helper() + db, err := database.NewDB(database.Config{DatabasePath: filepath.Join(t.TempDir(), "db")}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + return db +} + +// A superseded derived release ID takes this node's own records and publish +// state with it, in every network, and nothing else. +func TestContentIndexWithdrawRemovesOwnRecordsAndState(t *testing.T) { + db := testDB(t) + conn, d := db.Connection(), db.Dialect() + recs := database.NewPeernetRecordRepository(conn, d) + state := database.NewPeernetPublishStateRepository(conn, d) + ctx := context.Background() + now := time.Now() + put := func(nw, rel, hash string, source int) { + t.Helper() + _, err := recs.Insert(ctx, database.PeernetRecord{ + NetworkID: nw, Kind: database.PeernetRecordKindRelease, ContentHash: hash, ReleaseID: rel, PublisherNodeID: hx("9"), + CreatedAt: now.Add(-time.Minute), ExpiresAt: now.Add(time.Hour), Source: source, Title: "t", Payload: []byte("p"), + ReleaseScheme: database.PeernetSchemeFiles, + }, database.PeernetRecordLimits{}) + if err != nil { + t.Fatal(err) + } + if err := state.Put(ctx, database.PeernetPublishState{NetworkID: nw, Kind: database.PeernetRecordKindRelease, ReleaseID: rel, PublishedAt: now}); err != nil { + t.Fatal(err) + } + } + put(hx("a"), hx("1"), hx("c"), database.PeernetRecordSourceLocal) + put(hx("b"), hx("1"), hx("d"), database.PeernetRecordSourceLocal) + put(hx("a"), hx("2"), hx("e"), database.PeernetRecordSourceLocal) + + c := &contentIndex{deps: Deps{Records: recs, PublishState: state}} + c.withdraw(ctx, []string{hx("1")}) + + for _, nw := range []string{hx("a"), hx("b")} { + if _, ok, _ := state.Get(ctx, nw, database.PeernetRecordKindRelease, hx("1"), ""); ok { + t.Errorf("publish state of the superseded release survived in %s", nw[:4]) + } + } + if _, ok, _ := state.Get(ctx, hx("a"), database.PeernetRecordKindRelease, hx("2"), ""); !ok { + t.Error("another release's state was removed") + } + pa, _ := recs.ListOwnSince(ctx, hx("a"), 0, 10) + pb, _ := recs.ListOwnSince(ctx, hx("b"), 0, 10) + if len(pa.Rows) != 1 || len(pb.Rows) != 0 { + t.Errorf("own records left: network a %d (want 1), network b %d (want 0)", len(pa.Rows), len(pb.Rows)) + } +} + +// Writing metadata for an old-format import gives its files a derived release +// ID without a restart, and growing the import supersedes the earlier ID. +func TestContentIndexDerivesIncrementally(t *testing.T) { + db := testDB(t) + repo := database.NewPeernetContentRepository(db.Connection(), db.Dialect()) + ms := metadata.NewMetadataService(t.TempDir()) + deps := Deps{Content: repo, Metadata: NewMetadataHost(ms), deriveDebounce: 20 * time.Millisecond} + ctx, cancel := context.WithCancel(context.Background()) + c := startContentIndex(ctx, deps) + if c == nil { + t.Fatal("no content index") + } + t.Cleanup(func() { cancel(); c.stop() }) + waitFor(t, "the initial pass", func() bool { return !c.running.Load() }) + + write := func(p string, ids ...string) { + t.Helper() + segs := make([]*metapb.SegmentData, len(ids)) + for i, id := range ids { + segs[i] = &metapb.SegmentData{Id: id, SegmentSize: 10, EndOffset: 9} + } + m := &metapb.FileMetadata{FileSize: 10, SegmentData: segs, SourceNzbPath: "/data/.nzbs/5/x.nzb.gz"} + if err := ms.WriteFileMetadata(p, m); err != nil { + t.Fatal(err) + } + } + releaseOf := func(p string) string { + e, _, _ := repo.GetByPath(ctx, p) + return e.ReleaseID + } + write("c/r/a.mkv", "a1") + waitFor(t, "first file derived", func() bool { return releaseOf("c/r/a.mkv") != "" }) + first := releaseOf("c/r/a.mkv") + e, _, _ := repo.GetByPath(ctx, "c/r/a.mkv") + if e.ReleaseScheme != database.PeernetSchemeFiles { + t.Fatalf("scheme %d, want files", e.ReleaseScheme) + } + write("c/r/b.mkv", "b1") + waitFor(t, "second file joins the release", func() bool { + return releaseOf("c/r/b.mkv") != "" && releaseOf("c/r/a.mkv") == releaseOf("c/r/b.mkv") + }) + if releaseOf("c/r/a.mkv") == first { + t.Fatal("the release ID did not change when the file set did") + } +} + +func waitFor(t *testing.T, what string, ok func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if ok() { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} diff --git a/internal/peernet/service/frontend_labels_test.go b/internal/peernet/service/frontend_labels_test.go new file mode 100644 index 000000000..03fc0ba7f --- /dev/null +++ b/internal/peernet/service/frontend_labels_test.go @@ -0,0 +1,82 @@ +package service + +import ( + "os" + "regexp" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The status page shows these machine keys with plain-language labels. The +// frontend test lists every key the backend can emit; this test keeps those +// lists equal to the keys found in the Go sources, so a new backend key without +// a label (or a stale entry) fails here and not silently in the UI. + +func goStrings(t *testing.T, file, pattern string) []string { + t.Helper() + src, err := os.ReadFile(file) + require.NoError(t, err) + var out []string + for _, m := range regexp.MustCompile(pattern).FindAllStringSubmatch(string(src), -1) { + if m[1] != "" { + out = append(out, m[1]) + } + } + return out +} + +func tsList(t *testing.T, name string) []string { + t.Helper() + src, err := os.ReadFile("../../../frontend/tests/peernetStats.test.ts") + require.NoError(t, err) + m := regexp.MustCompile(`(?s)const ` + name + ` = \[(.*?)\];`).FindStringSubmatch(string(src)) + require.NotNil(t, m, "list %s not found in the frontend test", name) + var out []string + for _, k := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(m[1], -1) { + out = append(out, k[1]) + } + return out +} + +func sorted(in ...[]string) []string { + seen := map[string]bool{} + var out []string + for _, l := range in { + for _, k := range l { + if !seen[k] { + seen[k] = true + out = append(out, k) + } + } + } + sort.Strings(out) + return out +} + +func TestFrontendLabelListsMatchTheBackendKeys(t *testing.T) { + const peernet = "../" + for name, backend := range map[string][]string{ + "BACKEND_REJECTION_KEYS": goStrings(t, peernet+"recsync/reasons.go", `Reason\w+:\s+"([a-z0-9_]+)",`), + "BACKEND_SKIP_KEYS": sorted( + goStrings(t, peernet+"publish/publish.go", `Skip\w+\s+SkipReason = "([a-z0-9_]*)"`), + goStrings(t, "publishing.go", `\bskip\w+\s+= "([a-z0-9_]+)"`)), + "BACKEND_NOT_SHAREABLE_KEYS": sorted( + goStrings(t, peernet+"bundle/build.go", `Reason\w+\s+ShareReason = "([a-z0-9_]+)"`), + goStrings(t, peernet+"bundlex/builder.go", `Reason\w+\s+= "([a-z0-9_]+)"`), + []string{"other"}), + "BACKEND_SERVE_REFUSED_KEYS": sorted( + goStrings(t, peernet+"bundlex/serve.go", `(?:deny\(w|refuse)\(?,?\s*"([a-z0-9_]+)"`), + []string{"aborted"}), + "BACKEND_FETCH_FAILED_KEYS": goStrings(t, peernet+"bundlex/fetch.go", `\bfail\w+\s+= "([a-z0-9_]+)"`), + "BACKEND_IMPORT_REJECT_KEYS": goStrings(t, peernet+"bundleimport/types.go", `Reason\w+\s+Reason = "([a-z0-9_]+)"`), + } { + t.Run(strings.ToLower(name), func(t *testing.T) { + require.NotEmpty(t, backend) + assert.Equal(t, sorted(backend), sorted(tsList(t, name))) + }) + } +} diff --git a/internal/peernet/service/hints.go b/internal/peernet/service/hints.go index 5de57e15a..6aad41fb4 100644 --- a/internal/peernet/service/hints.go +++ b/internal/peernet/service/hints.go @@ -2,6 +2,7 @@ package service import ( "context" + "strconv" "sync" "sync/atomic" "time" @@ -34,6 +35,9 @@ const ( type HintAdvisor interface { HealthAdvice(ctx context.Context, releaseID, fileID string, localBackbones []string, now time.Time) (hints.Advice, error) NetworkHealthy(ctx context.Context, releaseIDs ...string) (map[string]hints.NetworkHealth, error) + // NetworkHealthyFiles labels releases from file-level evidence; the map is + // release ID to the File IDs of its main files. + NetworkHealthyFiles(ctx context.Context, releases map[string][]string) (map[string]hints.NetworkHealth, error) // SpendRecheck pays for a check moved forward on unhealthy evidence. SpendRecheck(releaseID, fileID string, evidenceAt, now time.Time) bool } @@ -71,10 +75,11 @@ type hintsHub struct { recheckScheduled int64 lookupErrors int64 cache map[string]labelEntry + candCache map[string]candEntry } func newHintsHub() *hintsHub { - return &hintsHub{given: map[string]int64{}, cache: map[string]labelEntry{}} + return &hintsHub{given: map[string]int64{}, cache: map[string]labelEntry{}, candCache: map[string]candEntry{}} } func (h *hintsHub) add(f func(*hintsHub)) { @@ -155,9 +160,11 @@ func (s *Service) HealthAdvice(ctx context.Context, filePath string) hints.Advic hub.lookupError() return hints.Advice{} } - if !ok || entry.ReleaseID == "" { + if !ok { return hints.Advice{} } + // Evidence is found by File ID, so a file with no Release ID still gets + // advice. advice, err := src.Advisor.HealthAdvice(ctx, entry.ReleaseID, entry.FileID, src.Backbones(), time.Now()) if err != nil || ctx.Err() != nil { hub.lookupError() @@ -181,9 +188,11 @@ func (s *Service) NotePeerAdviceApplied(kind hints.AdviceKind) { // NetworkHealthyPaths returns the paths (of those given, at most // database.PeernetMaxPathsPerQuery) whose release is healthy on the network. -// The match is exact by release ID from the content index; a path without one -// is never labelled. It is one bounded batch lookup, cached for a short time, -// and fails open to an empty result. +// The release is the one the content index has for the path (a path without +// one is never labelled) and it is healthy when every one of its video files +// has qualifying evidence, matched by File ID (see hints.NetworkHealthyFiles). +// It is one bounded batch lookup (one release-to-files query, one advisor +// call), cached for a short time, and fails open to an empty result. func (s *Service) NetworkHealthyPaths(ctx context.Context, paths []string) map[string]bool { out := map[string]bool{} n, src := s.hintContext() @@ -201,7 +210,7 @@ func (s *Service) NetworkHealthyPaths(ctx context.Context, paths []string) map[s hub.lookupError() return out } - healthy, err := s.releaseLabels(ctx, src, rels) + healthy, err := s.releaseLabels(ctx, n, src, rels) if err != nil { hub.lookupError() return out @@ -215,7 +224,7 @@ func (s *Service) NetworkHealthyPaths(ctx context.Context, paths []string) map[s } // releaseLabels answers per release, from the short cache where it can. -func (s *Service) releaseLabels(ctx context.Context, src *HintSources, rels map[string]string) (map[string]bool, error) { +func (s *Service) releaseLabels(ctx context.Context, n *node, src *HintSources, rels map[string]string) (map[string]bool, error) { hub, now := s.deps.hub, time.Now() res := map[string]bool{} var missing []string @@ -236,7 +245,18 @@ func (s *Service) releaseLabels(ctx context.Context, src *HintSources, rels map[ if len(missing) == 0 { return res, nil } - got, err := src.Advisor.NetworkHealthy(ctx, missing...) + // The label is about the release's main files: look them up in one query, + // ask for one more than the label can use so a release with too many is + // recognised. + files, err := n.deps.Content.VideoFilesOfReleases(ctx, missing, hints.MaxLabelFilesPerRelease) + if err != nil { + return nil, err + } + asked := make(map[string][]string, len(missing)) + for _, rel := range missing { + asked[rel] = files[rel] + } + got, err := src.Advisor.NetworkHealthyFiles(ctx, asked) if err != nil { return nil, err } @@ -320,13 +340,12 @@ func (n *node) recheckNetwork(ctx context.Context, src *HintSources, network str if considered >= recheckFilesPerRef { break } - if e.ReleaseID != ref.ReleaseID { - continue - } + // Evidence is about the file, whatever release ID it was + // published under: every local copy of the file qualifies. considered++ // Asking is read-only: the budget is spent below, and only when // a check really was moved forward. - advice, err := src.Advisor.HealthAdvice(ctx, ref.ReleaseID, ref.FileID, src.Backbones(), now) + advice, err := src.Advisor.HealthAdvice(ctx, e.ReleaseID, ref.FileID, src.Backbones(), now) if err != nil { hub.lookupError() return false @@ -340,7 +359,7 @@ func (n *node) recheckNetwork(ctx context.Context, src *HintSources, network str failed = true continue } - if moved && src.Advisor.SpendRecheck(ref.ReleaseID, ref.FileID, advice.EvidenceAt, now) { + if moved && src.Advisor.SpendRecheck(e.ReleaseID, ref.FileID, advice.EvidenceAt, now) { hub.add(func(h *hintsHub) { h.recheckScheduled++ }) } } @@ -388,3 +407,120 @@ func (n *node) clearHintCursor(id netid.NetworkID) { delete(n.hintCursor, id.String()) delete(n.hintRetries, id.String()) } + +// PeerCandidate is one release other nodes announced for a media key that this +// node can fetch a bundle of and does not have. Title is peer-supplied data: +// render it as text only. +type PeerCandidate struct { + ReleaseID string + Title string + TotalSize int64 + // Healthy: healthy on the network (see hints.NetworkHealth). + Healthy bool +} + +// candidateAdvisor is the optional part of the advisor that lists release +// records for a media key (*hints.Advisor has it). +type candidateAdvisor interface { + Candidates(ctx context.Context, imdbID string, season, episode *int) ([]hints.Candidate, error) +} + +const ( + peerCandidatesTimeout = 3 * time.Second + peerCandidatesTTL = 15 * time.Second + peerCandidatesCache = 64 + // peerCandidatesScan bounds how many records one lookup checks for a + // fetchable offer; peerCandidatesMax how many it returns. + peerCandidatesScan = 20 + peerCandidatesMax = 10 +) + +type candEntry struct { + list []PeerCandidate + at time.Time +} + +// PeerCandidates lists releases other nodes announced for a media key (IMDb ID, +// and season and episode for series; nil for a movie) that have a bundle this +// node can fetch from a verified peer and that this node does not already have. +// Healthy-on-the-network releases come first, then newest; at most ten. It is +// one bounded lookup with a short timeout and a short cache, and fails open: on +// any error, with the feature's consumers off or with no receiving network it +// returns nothing. +func (s *Service) PeerCandidates(ctx context.Context, imdbID string, season, episode *int) []PeerCandidate { + n, src := s.hintContext() + if n == nil || src == nil || n.fetcher == nil || n.deps.Content == nil || len(s.ReceivingNetworks()) == 0 { + return nil + } + adv, ok := src.Advisor.(candidateAdvisor) + if !ok { + return nil + } + key := candidateKey(imdbID, season, episode) + hub := s.deps.hub + hub.mu.Lock() + if e, ok := hub.candCache[key]; ok && time.Since(e.at) < peerCandidatesTTL { + hub.mu.Unlock() + return append([]PeerCandidate(nil), e.list...) + } + hub.mu.Unlock() + + ctx, cancel := context.WithTimeout(ctx, peerCandidatesTimeout) + defer cancel() + found, err := adv.Candidates(ctx, imdbID, season, episode) + if err != nil { + hub.lookupError() + return nil + } + var rels []string + for _, c := range found { + if !c.Own { + rels = append(rels, c.ReleaseID) + } + } + var local map[string]bool + if len(rels) > 0 { + if local, err = n.deps.Content.StillIndexed(ctx, rels); err != nil { + hub.lookupError() + return nil + } + } + var out []PeerCandidate + scanned := 0 + for _, c := range found { + if c.Own || local[c.ReleaseID] || len(out) >= peerCandidatesMax || scanned >= peerCandidatesScan { + continue + } + scanned++ + offers, err := n.fetcher.Offers(ctx, c.ReleaseID) + if err != nil { + hub.lookupError() + return nil + } + for _, o := range offers { + if o.Fetchable && !o.Own { + out = append(out, PeerCandidate{ReleaseID: c.ReleaseID, Title: c.Title, TotalSize: c.TotalSize, Healthy: c.Health.Healthy}) + break + } + } + } + hub.mu.Lock() + if len(hub.candCache) >= peerCandidatesCache { + clear(hub.candCache) + } + hub.candCache[key] = candEntry{list: append([]PeerCandidate(nil), out...), at: time.Now()} + hub.mu.Unlock() + return out +} + +func candidateKey(imdbID string, season, episode *int) string { + k := imdbID + for _, v := range []*int{season, episode} { + if v == nil { + k += "|-" + } else { + k += "|" + strconv.Itoa(*v) + } + } + return k +} diff --git a/internal/peernet/service/hints_hub_test.go b/internal/peernet/service/hints_hub_test.go index 3dd46a045..9ec3003b3 100644 --- a/internal/peernet/service/hints_hub_test.go +++ b/internal/peernet/service/hints_hub_test.go @@ -23,6 +23,9 @@ type fakeAdvisor struct { gotBB []string spends int denied bool // SpendRecheck refuses (budget gone) + // gotRel and gotReleases are what the last advice / label call was asked. + gotRel string + gotReleases map[string][]string } func (f *fakeAdvisor) SpendRecheck(_, _ string, _, _ time.Time) bool { @@ -37,7 +40,7 @@ func (f *fakeAdvisor) spendCount() int { f.mu.Lock(); defer f.mu.Unlock(); retur func (f *fakeAdvisor) HealthAdvice(ctx context.Context, rel, file string, bb []string, _ time.Time) (hints.Advice, error) { f.mu.Lock() f.calls++ - f.gotBB = bb + f.gotBB, f.gotRel = bb, rel d, a, err := f.delay, f.advice, f.err f.mu.Unlock() if d > 0 { @@ -64,6 +67,21 @@ func (f *fakeAdvisor) NetworkHealthy(_ context.Context, ids ...string) (map[stri return out, nil } +func (f *fakeAdvisor) NetworkHealthyFiles(_ context.Context, releases map[string][]string) (map[string]hints.NetworkHealth, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.nhCalls++ + f.gotReleases = releases + if f.err != nil { + return nil, f.err + } + out := map[string]hints.NetworkHealth{} + for rel := range releases { + out[rel] = f.healthy[rel] + } + return out, nil +} + func (f *fakeAdvisor) callCount() int { f.mu.Lock(); defer f.mu.Unlock(); return f.calls } type fakeScheduler struct { @@ -405,3 +423,73 @@ func TestStatusReportsBudget(t *testing.T) { t.Fatalf("budget remaining = %d", got) } } + +// A file with no Release ID of any kind (its import could not be grouped) still +// has a File ID, and evidence is found by file: it gets advice. +func TestHealthAdviceForFileWithoutReleaseID(t *testing.T) { + ctx := context.Background() + adv := &fakeAdvisor{advice: hints.Advice{Kind: hints.AdviceReduceSampleToFloor}} + f := hubFixture(t, adv, nil, hints.NewBudget(10)) + const bare = "movies/Bare/bare.mkv" + if err := f.content.Upsert(ctx, database.PeernetContentEntry{FilePath: bare, FileID: strings.Repeat("7", 64)}); err != nil { + t.Fatal(err) + } + if got := f.svc.HealthAdvice(ctx, bare); got.Kind != hints.AdviceReduceSampleToFloor { + t.Fatalf("advice for a file with only a file ID = %v", got.Kind) + } + if adv.gotRel != "" { + t.Fatalf("release passed for a file without one: %q", adv.gotRel) + } +} + +// Evidence is about the file: an attestation published under another release +// ID (an NZB-based one, for a file this node knows by its file set) still moves +// this node's check forward. +func TestRecheckPassFindsEvidenceUnderAnotherReleaseID(t *testing.T) { + ctx := context.Background() + adv := &fakeAdvisor{advice: hints.Advice{Kind: hints.AdviceRecheckSooner}} + sch := &fakeScheduler{} + f := hubFixture(t, adv, sch, hints.NewBudget(10)) + insertUnhealthy(t, f, strings.Repeat("e", 64), pubFile, 1) // same file, other release ID + f.node(t).maintainHints(ctx) + if got := sch.got(); len(got) != 1 || got[0] != pubPath { + t.Fatalf("scheduled: %v", got) + } +} + +// The release label asks about the release's video files only, in one batch for +// every release on the page. +func TestNetworkHealthyPathsAsksFileLevelInOneBatch(t *testing.T) { + ctx := context.Background() + adv := &fakeAdvisor{healthy: map[string]hints.NetworkHealth{pubRelease: {Healthy: true}}} + f := hubFixture(t, adv, nil, hints.NewBudget(10)) + hx := func(c string) string { return strings.Repeat(c, 64) } + rows := []database.PeernetContentEntry{ + {FilePath: "movies/Some.Movie.2020/movie.nfo", FileID: hx("1"), ReleaseID: pubRelease}, + {FilePath: "movies/Some.Movie.2020/extra.MP4", FileID: hx("2"), ReleaseID: pubRelease}, + {FilePath: "movies/Docs/a.pdf", FileID: hx("3"), ReleaseID: hx("8")}, // a release with no video + {FilePath: "movies/Other/o.mkv", FileID: hx("4"), ReleaseID: hx("9"), ReleaseScheme: database.PeernetSchemeFiles}, + } + for _, r := range rows { + if err := f.content.Upsert(ctx, r); err != nil { + t.Fatal(err) + } + } + got := f.svc.NetworkHealthyPaths(ctx, []string{pubPath, "movies/Docs/a.pdf", "movies/Other/o.mkv"}) + if len(got) != 1 || !got[pubPath] { + t.Fatalf("labels: %v", got) + } + if adv.nhCalls != 1 { + t.Fatalf("%d advisor calls for one page, want 1", adv.nhCalls) + } + files := adv.gotReleases[pubRelease] + if len(files) != 2 || files[0] != pubFile && files[1] != pubFile { + t.Fatalf("files asked for the release: %v (want its two video files)", files) + } + if _, asked := adv.gotReleases[hx("8")]; asked && len(adv.gotReleases[hx("8")]) != 0 { + t.Fatalf("a release with no video file was asked about: %v", adv.gotReleases[hx("8")]) + } + if len(adv.gotReleases[hx("9")]) != 1 { + t.Fatalf("derived release not asked about its file: %v", adv.gotReleases[hx("9")]) + } +} diff --git a/internal/peernet/service/index.go b/internal/peernet/service/index.go index bba334125..07cbb35a1 100644 --- a/internal/peernet/service/index.go +++ b/internal/peernet/service/index.go @@ -6,6 +6,7 @@ import ( "log/slog" "sync" "sync/atomic" + "time" "github.com/kipsilabs/altmount/internal/peernet/contentid" ) @@ -24,8 +25,23 @@ type contentIndex struct { scanned atomic.Int64 indexed atomic.Int64 skipped atomic.Int64 + // oversize is the number of files the last full derive pass left without + // a release ID because their group is over the cap. + oversize atomic.Int64 + + deriveKick chan struct{} // a group changed: derive it soon + deriveDebounce time.Duration + deriveMaxWait time.Duration } +// Timing of the incremental derive: wait for a quiet period so a burst of +// metadata writes (an import, a health sweep) is derived once, but never +// longer than the maximum. +const ( + defaultDeriveDebounce = 3 * time.Second + defaultDeriveMaxWait = 30 * time.Second +) + // startContentIndex hooks the metadata observer and starts the index worker. // Observer events go through a bounded queue served by one worker, so the // import write path never waits for hashing or the database; when the queue @@ -37,16 +53,23 @@ func startContentIndex(parent context.Context, deps Deps) *contentIndex { return nil } ctx, cancel := context.WithCancel(parent) - c := &contentIndex{deps: deps, cancel: cancel, kick: make(chan struct{}, 1)} + c := &contentIndex{deps: deps, cancel: cancel, kick: make(chan struct{}, 1), deriveKick: make(chan struct{}, 1), + deriveDebounce: defaultDeriveDebounce, deriveMaxWait: defaultDeriveMaxWait} + if deps.deriveDebounce > 0 { + c.deriveDebounce = deps.deriveDebounce + } idx := contentid.NewIndexer(deps.Metadata, deps.Content, - contentid.WithProgress(func(r contentid.BackfillResult) { c.store(r) })) + contentid.WithProgress(func(r contentid.BackfillResult) { c.store(r) }), + contentid.WithOrphanHandler(c.withdraw), + contentid.WithDirtyNotify(c.requestDerive)) obs := contentid.NewAsyncObserver(contentid.NewObserver(idx), 0, c.requestReindex) - c.wg.Add(2) + c.wg.Add(3) go func() { defer c.wg.Done() obs.Run(ctx) }() go c.worker(ctx, idx) + go c.deriveLoop(ctx, idx) deps.Metadata.SetFileObserver(obs) return c } @@ -64,6 +87,90 @@ func (c *contentIndex) requestReindex() { } } +func (c *contentIndex) requestDerive() { + select { + case c.deriveKick <- struct{}{}: + default: + } +} + +// deriveLoop runs the incremental derive pass (see contentid.DeriveDirty) +// after a quiet period following a change to an import group. +func (c *contentIndex) deriveLoop(ctx context.Context, idx *contentid.Indexer) { + defer c.wg.Done() + for { + select { + case <-ctx.Done(): + return + case <-c.deriveKick: + } + first := time.Now() + timer := time.NewTimer(c.deriveDebounce) + quiet: + for { + select { + case <-ctx.Done(): + timer.Stop() + return + case <-c.deriveKick: + if time.Since(first) >= c.deriveMaxWait { + break quiet + } + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + timer.Reset(c.deriveDebounce) + case <-timer.C: + break quiet + } + } + timer.Stop() + if _, err := idx.DeriveDirty(ctx); err != nil && !errors.Is(err, context.Canceled) { + slog.ErrorContext(ctx, "Peernet incremental release id derivation failed", "error", err) + } + } +} + +// withdraw retracts what this node published under library-derived release IDs +// that no file carries any more (the group gained or lost files, or vanished): +// its own records are deleted from every network and the publish state +// forgotten, so a stale release identity is neither served nor re-announced. +// Copies other nodes already hold cannot be recalled and expire on their own. +func (c *contentIndex) withdraw(ctx context.Context, releaseIDs []string) { + if c.deps.onWithdraw != nil { + c.deps.onWithdraw(releaseIDs) + } + recs, state := c.deps.Records, c.deps.PublishState + if recs == nil && state == nil { + return + } + var networks []string + if recs != nil { + var err error + if networks, err = recs.NetworkIDs(ctx); err != nil { + slog.WarnContext(ctx, "Peernet could not list networks to withdraw superseded releases", "error", err) + return + } + } + for _, id := range releaseIDs { + for _, nw := range networks { + if _, err := recs.DeleteOwnRelease(ctx, nw, id); err != nil { + slog.WarnContext(ctx, "Peernet could not withdraw a superseded release", "error", err) + } + } + if state != nil { + for _, nw := range networks { + if err := state.DeleteRelease(ctx, nw, id); err != nil { + slog.WarnContext(ctx, "Peernet could not forget a superseded release", "error", err) + } + } + } + } +} + // stop unhooks the observer and waits for the worker. func (c *contentIndex) stop() { c.deps.Metadata.SetFileObserver(nil) @@ -86,6 +193,13 @@ func (c *contentIndex) worker(ctx context.Context, idx *contentid.Indexer) { if err != nil && !errors.Is(err, context.Canceled) { slog.ErrorContext(ctx, "Peernet content index backfill failed", "error", err) } + // Older imports: give each import group its library-derived release ID + // from the index alone (no metadata is read again). + if dres, err := idx.DeriveAll(ctx); err != nil && !errors.Is(err, context.Canceled) { + slog.ErrorContext(ctx, "Peernet library release id derivation failed", "error", err) + } else if err == nil { + c.oversize.Store(int64(dres.OversizeFiles)) + } c.running.Store(false) select { case <-ctx.Done(): @@ -96,16 +210,31 @@ func (c *contentIndex) worker(ctx context.Context, idx *contentid.Indexer) { } // LibraryStatus says how much of the library can be shared. A file can be -// shared when it has a Release ID; one with only a File ID was imported before -// the release's NZB was kept (or from a rebuilt store), and one that could not -// be identified at all is not indexable. +// shared when it has a Release ID: either one from its release's NZB +// ("nzb"), or one derived from the files of its import ("files", for imports +// that predate the NZB store). A file with only a File ID is in an import +// whose files could not be grouped, and one that could not be identified at +// all is not indexable. type LibraryStatus struct { // Files is every library file the index knows about plus those that could // not be identified. - Files int - WithReleaseID int - FileIDOnly int - NotIndexable int + Files int + // WithReleaseID is the files that can be shared: WithNZBReleaseID plus + // WithDerivedReleaseID. + WithReleaseID int + WithNZBReleaseID int + WithDerivedReleaseID int + // FileIDOnly are indexed files with no Release ID; NoSourceNZB, GroupTooLarge + // and Pending say why (they add up to FileIDOnly). + FileIDOnly int + // NotIndexable files could not be identified at all. + NotIndexable int + // NoSourceNZB: the metadata does not say which import the file came from. + NoSourceNZB int + // GroupTooLarge: the import has more files than a derived release may have. + GroupTooLarge int + // Pending: the file is in a group the derive pass has not reached yet. + Pending int } // library reads the index counts. It never fails the status: on a read error @@ -116,10 +245,17 @@ func (c *contentIndex) library(ctx context.Context) LibraryStatus { slog.WarnContext(ctx, "Peernet library counts unavailable", "error", err) } skipped := int(c.skipped.Load()) + idOnly := counts.Files - counts.WithReleaseID + tooLarge := min(int(c.oversize.Load()), max(idOnly-counts.NoSourceGroup, 0)) return LibraryStatus{ - Files: counts.Files + skipped, - WithReleaseID: counts.WithReleaseID, - FileIDOnly: counts.Files - counts.WithReleaseID, - NotIndexable: skipped, + Files: counts.Files + skipped, + WithReleaseID: counts.WithReleaseID, + WithNZBReleaseID: counts.WithNZBReleaseID, + WithDerivedReleaseID: counts.WithDerivedReleaseID, + FileIDOnly: idOnly, + NotIndexable: skipped, + NoSourceNZB: counts.NoSourceGroup, + GroupTooLarge: tooLarge, + Pending: max(idOnly-counts.NoSourceGroup-tooLarge, 0), } } diff --git a/internal/peernet/service/library_status_test.go b/internal/peernet/service/library_status_test.go index d4a6a9f00..25892a568 100644 --- a/internal/peernet/service/library_status_test.go +++ b/internal/peernet/service/library_status_test.go @@ -19,9 +19,11 @@ func TestLibraryStatusCountsShareableFiles(t *testing.T) { hex := func(c string) string { return strings.Repeat(c, 64) } ctx := context.Background() for _, e := range []database.PeernetContentEntry{ - {FilePath: "a.mkv", FileID: hex("a"), ReleaseID: hex("1")}, - {FilePath: "b.mkv", FileID: hex("b"), ReleaseID: hex("1")}, - {FilePath: "c.mkv", FileID: hex("c")}, + {FilePath: "a.mkv", FileID: hex("a"), ReleaseID: hex("1"), ReleaseScheme: database.PeernetSchemeNZB}, + {FilePath: "b.mkv", FileID: hex("b"), ReleaseID: hex("2"), ReleaseScheme: database.PeernetSchemeFiles, GroupKey: hex("d")}, + {FilePath: "c.mkv", FileID: hex("c")}, // no source NZB in its metadata + {FilePath: "d.mkv", FileID: hex("d"), GroupKey: hex("e")}, // in a group over the cap + {FilePath: "e.mkv", FileID: hex("e"), GroupKey: hex("f")}, // waiting for the next derive pass } { if err := repo.Upsert(ctx, e); err != nil { t.Fatal(err) @@ -29,8 +31,10 @@ func TestLibraryStatusCountsShareableFiles(t *testing.T) { } c := &contentIndex{deps: Deps{Content: repo}} c.skipped.Store(2) + c.oversize.Store(1) got := c.library(ctx) - want := LibraryStatus{Files: 5, WithReleaseID: 2, FileIDOnly: 1, NotIndexable: 2} + want := LibraryStatus{Files: 7, WithReleaseID: 2, WithNZBReleaseID: 1, WithDerivedReleaseID: 1, FileIDOnly: 3, NotIndexable: 2, + NoSourceNZB: 1, GroupTooLarge: 1, Pending: 1} if got != want { t.Fatalf("library = %+v, want %+v", got, want) } diff --git a/internal/peernet/service/node.go b/internal/peernet/service/node.go index b85b204f5..cfd8a0acc 100644 --- a/internal/peernet/service/node.go +++ b/internal/peernet/service/node.go @@ -14,6 +14,7 @@ import ( "time" "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/peernet/bundlex" "github.com/kipsilabs/altmount/internal/peernet/discovery" "github.com/kipsilabs/altmount/internal/peernet/identity" "github.com/kipsilabs/altmount/internal/peernet/membership" @@ -48,6 +49,9 @@ type node struct { disc *discovery.Service rx *recsync.Exchange // nil when record storage is not configured pipe *pipeline // nil when publishing is not configured + builder *bundlex.Builder // nil without a full-metadata source + bsrv *bundlex.Server // nil unless bundles can be served + fetcher *bundlex.Fetcher // nil when record storage is not configured mem *membership.Manager kick chan struct{} // wakes the discovery loop maintKick chan struct{} // wakes the membership upkeep @@ -141,10 +145,12 @@ func startNode(parent context.Context, deps Deps, s Settings) (*node, error) { n.abort() return nil, fmt.Errorf("start record exchange: %w", err) } + n.startBuilder() if err := n.startPublishing(); err != nil { n.abort() return nil, fmt.Errorf("start publishing: %w", err) } + n.startBundleServe() n.reconcileFn = n.disc.Reconcile n.disc.AddSource(hostSource{mem: n.mem}) if s.AcceptIncoming { @@ -256,6 +262,11 @@ func (n *node) registerRoutes(srv *transport.Server) error { return err } } + if n.bsrv != nil { + if err := n.bsrv.RegisterRoutes(srv); err != nil { + return err + } + } return n.mem.RegisterHandlers(srv) } diff --git a/internal/peernet/service/peer_candidates_test.go b/internal/peernet/service/peer_candidates_test.go new file mode 100644 index 000000000..a8466dab1 --- /dev/null +++ b/internal/peernet/service/peer_candidates_test.go @@ -0,0 +1,129 @@ +package service + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/peernet/hints" +) + +const candIMDb = "tt0111161" + +// candidateFixture: a level-3 publisher whose release carries a media id, a +// level-1 receiver that synced its records and has a real advisor installed. +func candidateFixture(t *testing.T, pubLevel int) (*bundlePub, *instance) { + t.Helper() + a := newBundlePublisher(t, pubLevel, nil) + if _, err := a.db.Connection().Exec(`UPDATE import_history SET metadata = ? WHERE virtual_path = ?`, + `{"imdb_id":"`+candIMDb+`","type":"movie"}`, pubPath); err != nil { + t.Fatal(err) + } + a.publishHealthy(t) + b := bundleReceiver(t, a) + b.syncRound(t) + installAdvisor(t, b) + return a, b +} + +func installAdvisor(t *testing.T, b *instance) { + t.Helper() + conn, d := b.db.Connection(), b.db.Dialect() + adv, err := hints.New(hints.Config{Store: database.NewPeernetRecordRepository(conn, d), Peers: database.NewPeernetPeerRepository(conn, d), + Budget: hints.NewBudget(6), Networks: b.svc.ReceivingNetworks, SelfNodeID: b.svc.NodeID()}) + if err != nil { + t.Fatal(err) + } + b.svc.SetHintSources(HintSources{Advisor: realAdvisor{adv}, Budget: hints.NewBudget(6), Backbones: func() []string { return nil }}) +} + +// realAdvisor adapts *hints.Advisor to HintAdvisor for tests (the production +// wiring passes the real one, which already has every method). +type realAdvisor struct{ *hints.Advisor } + +func TestPeerCandidatesListsFetchableOffersOnly(t *testing.T) { + a, b := candidateFixture(t, 3) + got := b.svc.PeerCandidates(context.Background(), candIMDb, nil, nil) + if len(got) != 1 || got[0].ReleaseID != a.rel || got[0].Title == "" || got[0].TotalSize == 0 { + t.Fatalf("candidates %+v", got) + } + // Another media key finds nothing. + if got := b.svc.PeerCandidates(context.Background(), "tt0000001", nil, nil); len(got) != 0 { + t.Fatalf("unrelated media: %+v", got) + } + // A level 2 publisher announces no bundle: nothing to fetch, nothing offered. + _, b2 := candidateFixture(t, 2) + if got := b2.svc.PeerCandidates(context.Background(), candIMDb, nil, nil); len(got) != 0 { + t.Fatalf("a record without a bundle was offered: %+v", got) + } +} + +func TestPeerCandidatesSkipWhatIsAlreadyLocal(t *testing.T) { + a, b := candidateFixture(t, 3) + content := database.NewPeernetContentRepository(b.db.Connection(), b.db.Dialect()) + if err := content.Upsert(context.Background(), database.PeernetContentEntry{FilePath: "movies/x/m.mkv", FileID: hexOf('f'), ReleaseID: a.rel}); err != nil { + t.Fatal(err) + } + if got := b.svc.PeerCandidates(context.Background(), candIMDb, nil, nil); len(got) != 0 { + t.Fatalf("a release this node already has was offered: %+v", got) + } +} + +func TestPeerCandidatesFailOpenAndCache(t *testing.T) { + _, b := candidateFixture(t, 3) + // Not installed (feature consumers off): nothing, no panic. + b.svc.ClearHintSources() + if got := b.svc.PeerCandidates(context.Background(), candIMDb, nil, nil); got != nil { + t.Fatalf("without hint sources: %+v", got) + } + // An advisor failure is no candidate and no error. + b.svc.SetHintSources(HintSources{Advisor: failingCandidates{}, Budget: hints.NewBudget(1), Backbones: func() []string { return nil }}) + if got := b.svc.PeerCandidates(context.Background(), candIMDb, nil, nil); got != nil { + t.Fatalf("advisor error: %+v", got) + } + // Invalid media ids never reach the store. + installAdvisor(t, b) + if got := b.svc.PeerCandidates(context.Background(), "not-an-id", nil, nil); got != nil { + t.Fatalf("invalid id: %+v", got) + } + // A repeat inside the cache window does not ask the advisor again. + counting := &countingCandidates{realAdvisor: realAdvisor{}} + conn, d := b.db.Connection(), b.db.Dialect() + adv, _ := hints.New(hints.Config{Store: database.NewPeernetRecordRepository(conn, d), Peers: database.NewPeernetPeerRepository(conn, d), + Budget: hints.NewBudget(6), Networks: b.svc.ReceivingNetworks, SelfNodeID: b.svc.NodeID()}) + counting.realAdvisor = realAdvisor{adv} + b.svc.SetHintSources(HintSources{Advisor: counting, Budget: hints.NewBudget(6), Backbones: func() []string { return nil }}) + b.svc.PeerCandidates(context.Background(), candIMDb, nil, nil) + b.svc.PeerCandidates(context.Background(), candIMDb, nil, nil) + if counting.calls != 1 { + t.Fatalf("advisor asked %d times", counting.calls) + } +} + +type failingCandidates struct{ realAdvisor } + +func (failingCandidates) Candidates(context.Context, string, *int, *int) ([]hints.Candidate, error) { + return nil, errors.New("db down") +} + +type countingCandidates struct { + realAdvisor + calls int +} + +func (c *countingCandidates) Candidates(ctx context.Context, imdb string, s, e *int) ([]hints.Candidate, error) { + c.calls++ + return c.realAdvisor.Candidates(ctx, imdb, s, e) +} + +func hexOf(c byte) string { + b := make([]byte, 64) + for i := range b { + b[i] = c + } + return string(b) +} + +var _ = time.Second diff --git a/internal/peernet/service/peer_origin_test.go b/internal/peernet/service/peer_origin_test.go new file mode 100644 index 000000000..398a86ea3 --- /dev/null +++ b/internal/peernet/service/peer_origin_test.go @@ -0,0 +1,143 @@ +package service + +import ( + "context" + "errors" + "testing" + "time" +) + +const mediaMetadata = `{"imdb_id":"tt0133093","type":"series","season":2,"episode":5,"tmdbId":603}` + +type fakePeerOrigin struct { + from, passed bool + err error + asked []string +} + +func (f *fakePeerOrigin) PeerBundleStatus(_ context.Context, p string) (bool, bool, error) { + f.asked = append(f.asked, p) + return f.from, f.passed, f.err +} + +func originFixture(t *testing.T, po *fakePeerOrigin, extra ...func(*Deps)) *pubFixture { + t.Helper() + f := newPubInstance(t, level(2), func(d *Deps) { + d.PeerOrigin = po + for _, m := range extra { + m(d) + } + }) + f.seedFile(t) + f.start(t) + return f +} + +func TestPublishPeerBundleFileWaitsForItsFullCheck(t *testing.T) { + for _, tc := range []struct { + name string + po fakePeerOrigin + published bool + skip string + releaseState bool + }{ + {name: "local import", po: fakePeerOrigin{}, published: true}, + {name: "peer import, check pending", po: fakePeerOrigin{from: true}, skip: "peer_bundle_not_fully_checked"}, + {name: "peer import, check passed", po: fakePeerOrigin{from: true, passed: true}, published: true}, + {name: "status unreadable", po: fakePeerOrigin{err: errors.New("db down")}, skip: "peer_status_unknown"}, + } { + t.Run(tc.name, func(t *testing.T) { + po := tc.po + f := originFixture(t, &po) + f.svc.NotifyHealthy(healthyNotice()) + f.drain(t) + got := f.status(t).Publish + if tc.published && got.Published["release"] == 0 { + t.Fatalf("not published: %+v", got) + } + if !tc.published && (len(got.Published) != 0 || got.Skipped[tc.skip] != 1) { + t.Fatalf("want one skip %q and nothing published: %+v", tc.skip, got) + } + }) + } +} + +func TestReannounceSkipsPeerBundleFilesUntilVerified(t *testing.T) { + po := &fakePeerOrigin{passed: true, from: true} + lh := &fakeLocalHealth{statuses: map[string]string{pubPath: "healthy"}} + f := originFixture(t, po, func(d *Deps) { d.Health = lh }) + f.svc.NotifyHealthy(healthyNotice()) + f.drain(t) + awaitFirstUpkeep(t, f) + ageRelease(t, f, pubRelease) + po.from, po.passed = true, false + f.node(t).pipe.reannounce(context.Background(), 10) + if got := f.status(t).Publish.Reannounced; got != 0 { + t.Fatalf("re-announced a peer import that is not verified: %d", got) + } + po.err = errors.New("db down") + f.node(t).pipe.reannounce(context.Background(), 10) + if got := f.status(t).Publish.Reannounced; got != 0 { + t.Fatalf("re-announced with unreadable peer state: %d", got) + } + po.err, po.passed = nil, true + f.node(t).pipe.clearFailure(openID, pubRelease) // skip the back-off of the refused passes + f.node(t).pipe.reannounce(context.Background(), 10) + if got := f.status(t).Publish.Reannounced; got != 1 { + t.Fatalf("a verified peer import must re-announce: %d", got) + } +} + +// A peer's unverified media claim must not be signed by this node: whatever +// the full-check state, a peer-bundle file publishes its release without +// media identifiers; a local import keeps them. +func TestPublishNeverRepublishesPeerBundleMediaIDs(t *testing.T) { + for _, tc := range []struct { + name string + po fakePeerOrigin + wantMedia bool + }{ + {name: "local import", po: fakePeerOrigin{}, wantMedia: true}, + {name: "peer import, check passed", po: fakePeerOrigin{from: true, passed: true}}, + } { + t.Run(tc.name, func(t *testing.T) { + po := tc.po + f := newPubInstance(t, level(2), func(d *Deps) { d.PeerOrigin = &po }) + f.seedFile(t, func(s *seed) { s.metadata = mediaMetadata }) + f.start(t) + f.svc.NotifyHealthy(healthyNotice()) + f.drain(t) + rel, _ := ownRecords(t, f) + if rel == nil { + t.Fatalf("no release record: %+v", f.status(t).Publish) + } + if got := rel.Media.IMDb != "" || rel.Media.TMDB != 0 || rel.Media.Season != nil || rel.Media.Episode != nil; got != tc.wantMedia { + t.Fatalf("media published = %v, want %v: %+v", got, tc.wantMedia, rel.Media) + } + }) + } +} + +func TestReannounceNeverRepublishesPeerBundleMediaIDs(t *testing.T) { + po := &fakePeerOrigin{} + lh := &fakeLocalHealth{statuses: map[string]string{pubPath: "healthy"}} + f := newPubInstance(t, level(2), func(d *Deps) { d.PeerOrigin, d.Health = po, lh }) + f.seedFile(t, func(s *seed) { s.metadata = mediaMetadata }) + f.start(t) + f.svc.NotifyHealthy(healthyNotice()) + f.drain(t) + awaitFirstUpkeep(t, f) + if rel, _ := ownRecords(t, f); rel == nil || rel.Media.IMDb == "" { + t.Fatalf("a local import must publish its media ids: %+v", rel) + } + ageRelease(t, f, pubRelease) + time.Sleep(1100 * time.Millisecond) // record times have second precision: the new record must be newer + po.from, po.passed = true, true + f.node(t).pipe.reannounce(context.Background(), 10) + if got := f.status(t).Publish.Reannounced; got != 1 { + t.Fatalf("not re-announced: %d %+v", got, f.status(t).Publish) + } + if rel, _ := ownRecords(t, f); rel == nil || rel.Media.IMDb != "" || rel.Media.TMDB != 0 { + t.Fatalf("re-announce republished a peer's media ids: %+v", rel) + } +} diff --git a/internal/peernet/service/publishing.go b/internal/peernet/service/publishing.go index 443563111..73e123a7d 100644 --- a/internal/peernet/service/publishing.go +++ b/internal/peernet/service/publishing.go @@ -33,6 +33,9 @@ const ( // skipPatchStatusUnknown: whether a local PAR2 patch covers the file was // not determined (the check began before the observer was installed). skipPatchStatusUnknown = "patch_status_unknown" + // skipPeerStatusUnknown: whether the file came from a peer bundle could not + // be read, so it is not eligible. + skipPeerStatusUnknown = "peer_status_unknown" ) // LocalHealth is the read-only view of local file health the re-announce @@ -219,11 +222,20 @@ func (s stateStore) LastPublished(ctx context.Context, nw netid.NetworkID, kind if err != nil || !ok { return publish.State{}, false, err } - return publish.State{PublishedAt: row.PublishedAt, CheckedAt: row.CheckedAt, Verdict: publish.Verdict(row.Verdict)}, true, nil + st := publish.State{PublishedAt: row.PublishedAt, CheckedAt: row.CheckedAt, Verdict: publish.Verdict(row.Verdict), + BundleLevel: row.BundleLevel} + if id, err := contentid.ParseID(row.BundleID); err == nil { + st.BundleID = records.BundleID(id) + } + return st, true, nil } func (s stateStore) MarkPublished(ctx context.Context, nw netid.NetworkID, kind records.Kind, rel records.ReleaseID, file records.FileID, st publish.State) error { - return s.repo.Put(ctx, database.PeernetPublishState{ + bundleHex := "" + if st.BundleID != (records.BundleID{}) { + bundleHex = contentid.ID(st.BundleID).String() + } + return s.repo.Put(ctx, database.PeernetPublishState{BundleID: bundleHex, BundleLevel: st.BundleLevel, NetworkID: nw.String(), Kind: kindToDB(kind), ReleaseID: contentid.ID(rel).String(), FileID: fileHex(file), PublishedAt: st.PublishedAt, CheckedAt: st.CheckedAt, Verdict: int(st.Verdict), OriginCategory: st.OriginCategory, OriginIndexer: st.OriginIndexer, @@ -271,6 +283,10 @@ type pipelineConfig struct { // stamped with it and checked against real time, so it is never moved // apart from Now outside tests. PublisherNow func() time.Time + // Bundles supplies Bundle IDs at level 3; nil announces none. + Bundles publish.BundleSource + // PeerOrigin gates files that came from a peer bundle; nil means none did. + PeerOrigin PeerBundleOrigin } // pipeline turns healthy-check notices into published records. The health @@ -304,6 +320,7 @@ func newPipeline(cfg pipelineConfig) (*pipeline, error) { pub, err := publish.New(publish.Config{ Signer: cfg.Signer, Sink: cfg.Sink, Store: stateStore{repo: cfg.State}, Backbones: backboneAdapter{src: cfg.Backbones}, Networks: cfg.Networks, Exclusions: cfg.Exclusions, Now: cfg.PublisherNow, + Bundles: cfg.Bundles, // Unhealthy attestations stay off (PublishUnhealthy is false). }) if err != nil { @@ -403,6 +420,26 @@ func (p *pipeline) handle(ctx context.Context, n HealthyNotice) { TotalSegments: uint32(n.TotalSegments), SegmentsChecked: uint32(n.SegmentsChecked), MediaHeaderVerified: n.MediaHeaderVerified, PatchedByPAR2: n.PatchedByPAR2, } + // A file that came from a peer bundle is published only once the full + // local check it owes has passed (the importer's persistent state, not + // this check's size, is the truth: later sampled re-checks of a verified + // file publish like any local import). + if p.cfg.PeerOrigin != nil { + from, passed, err := p.cfg.PeerOrigin.PeerBundleStatus(ctx, n.FilePath) + if err != nil { + p.cfg.Log(ctx, "publish: peer bundle status", err) + p.skip(skipPeerStatusUnknown) + return + } + res.FromPeerBundle = from && !passed + res.FullCheckPassed = false + if from { + // The media ids of a peer's import are that peer's unverified + // claim; never sign them as this node's (cached info is shared, + // so strip on this copy). + info.Media = records.MediaIDs{} + } + } // A reduced check, or one whose patch status was never determined, may // still announce the release but is no evidence: release record only. switch { @@ -420,6 +457,9 @@ func (p *pipeline) handle(ctx context.Context, n HealthyNotice) { func (p *pipeline) count(ctx context.Context, outs []publish.Outcome, err error) { failed := false for _, o := range outs { + if o.BundleErr != nil { + p.fail(ctx, "publish: build bundle", o.BundleErr) + } switch { case o.Err != nil: failed = true @@ -484,7 +524,7 @@ func (p *pipeline) reannounce(ctx context.Context, limit int) { // reannounceOne re-announces one release and reports whether it went through. func (p *pipeline) reannounceOne(ctx context.Context, t publish.Tracked, relHex string) bool { - ok, err := p.locallyHealthy(ctx, relHex) + ok, fromPeer, err := p.locallyHealthy(ctx, relHex) if err != nil { p.cfg.Log(ctx, "publish: re-announce health lookup", err) return false @@ -503,7 +543,15 @@ func (p *pipeline) reannounceOne(ctx context.Context, t publish.Tracked, relHex } return false } + if fromPeer { + info.Media = records.MediaIDs{} // a peer's unverified claim is not re-signed + } o := p.pub.ReannounceRelease(ctx, t.Network, t.ReleaseID, info) + if o.BundleErr != nil { + // The bundle could not be built; nothing was sent. Back off before the next attempt. + p.fail(ctx, "publish: build bundle", o.BundleErr) + return false + } switch { case o.Err != nil: p.fail(ctx, "publish: re-announce", o.Err) @@ -520,17 +568,18 @@ func (p *pipeline) reannounceOne(ctx context.Context, t publish.Tracked, relHex // locallyHealthy reports whether the release is still indexed and each of its // indexed files has a healthy status. No health source, no files, too many -// files or any missing record is "not healthy". -func (p *pipeline) locallyHealthy(ctx context.Context, relHex string) (bool, error) { +// files or any missing record is "not healthy". fromPeer reports that any +// file came from a peer bundle (its media ids must not be re-signed). +func (p *pipeline) locallyHealthy(ctx context.Context, relHex string) (healthy, fromPeer bool, err error) { if p.cfg.Health == nil { - return false, nil + return false, false, nil } entries, err := p.cfg.Content.FindByReleaseID(ctx, relHex) if err != nil { - return false, fmt.Errorf("list release files: %w", err) + return false, false, fmt.Errorf("list release files: %w", err) } if len(entries) == 0 || len(entries) > maxReleaseFiles { - return false, nil + return false, false, nil } paths := make([]string, len(entries)) for i, e := range entries { @@ -538,14 +587,26 @@ func (p *pipeline) locallyHealthy(ctx context.Context, relHex string) (bool, err } st, err := p.cfg.Health.Statuses(ctx, paths) if err != nil { - return false, fmt.Errorf("local health: %w", err) + return false, false, fmt.Errorf("local health: %w", err) } for _, path := range paths { if st[path] != string(database.HealthStatusHealthy) { - return false, nil + return false, false, nil } } - return true, nil + if p.cfg.PeerOrigin != nil { + for _, path := range paths { + from, passed, err := p.cfg.PeerOrigin.PeerBundleStatus(ctx, path) + if err != nil { + return false, false, fmt.Errorf("peer bundle status: %w", err) + } + if from && !passed { + return false, false, nil + } + fromPeer = fromPeer || from + } + } + return true, fromPeer, nil } func backoffKey(nw netid.NetworkID, rel string) string { return nw.String() + "|" + rel } @@ -624,18 +685,26 @@ func (n *node) startPublishing() error { if n.rx == nil || d.Content == nil || d.Origins == nil || d.PublishState == nil || d.Metadata == nil { return nil } - p, err := newPipeline(pipelineConfig{ + var bundles publish.BundleSource + if n.builder != nil { + bundles = bundleSource{b: n.builder} + } + p, err := newPipeline(pipelineConfig{Bundles: bundles, Signer: nodeSigner{key: n.key}, Sink: exchangeSink{rx: n.rx}, Networks: func() []publish.Network { ids := n.rx.PublishNetworks() out := make([]publish.Network, 0, len(ids)) for _, id := range ids { - out = append(out, publish.Network{ID: id, Level: publish.MinLevel}) + lvl, ok := n.rx.NetworkLevel(id) + if !ok { + continue + } + out = append(out, publish.Network{ID: id, Level: lvl}) } return out }, Exclusions: d.Exclusions, Backbones: d.Backbones, - State: d.PublishState, Content: d.Content, Origins: d.Origins, Meta: d.Metadata, Health: d.Health, + State: d.PublishState, Content: d.Content, Origins: d.Origins, Meta: d.Metadata, Health: d.Health, PeerOrigin: d.PeerOrigin, QueueSize: d.publishQueueSize, Now: d.publishNow, PublisherNow: time.Now, Hold: d.beforePublish, Log: n.logChange, }) if err != nil { @@ -669,6 +738,7 @@ func (n *node) maintainPublishing(ctx context.Context, orphans bool) { n.logChange(ctx, "delete orphan publish state", err) } n.pipe.reannounce(ctx, reannounceBatch) + n.pipe.reannounceLevelChanges(ctx, reannounceBatch) } // retractExcluded deletes this node's own records and publish state of @@ -705,3 +775,35 @@ const ( reannounceBatch = 20 retractBatch = 50 ) + +// reannounceLevelChanges announces again, up to limit records per network, +// those published at a different bundle level than the network has now: a +// network moved to level 3 announces its bundles, one moved below 3 supersedes +// the records that carried a Bundle ID. A release that fails backs off like any +// re-announce. +func (p *pipeline) reannounceLevelChanges(ctx context.Context, limit int) { + now := p.cfg.Now() + for _, nw := range p.cfg.Networks() { + rows, err := p.cfg.State.ListBundleLevelMismatch(ctx, nw.ID.String(), nw.Level >= publish.BundleMinLevel, limit) + if err != nil { + p.cfg.Log(ctx, "publish: list bundle level changes", err) + return + } + p.cfg.Log(ctx, "publish: list bundle level changes", nil) + for _, r := range rows { + if ctx.Err() != nil { + return + } + rel, err := contentid.ParseID(r.ReleaseID) + if err != nil || p.backedOff(nw.ID, r.ReleaseID, now) { + continue + } + t := publish.Tracked{Network: nw.ID, Kind: records.KindRelease, ReleaseID: records.ReleaseID(rel)} + if p.reannounceOne(ctx, t, r.ReleaseID) { + p.clearFailure(nw.ID, r.ReleaseID) + } else { + p.noteFailure(nw.ID, r.ReleaseID, now) + } + } + } +} diff --git a/internal/peernet/service/publishing_derived_test.go b/internal/peernet/service/publishing_derived_test.go new file mode 100644 index 000000000..2e8dc38a3 --- /dev/null +++ b/internal/peernet/service/publishing_derived_test.go @@ -0,0 +1,100 @@ +package service + +import ( + "context" + "encoding/hex" + "testing" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/peernet/netid" + "github.com/kipsilabs/altmount/internal/peernet/records" +) + +const derivedPath2 = "movies/Some.Movie.2020/movie.nfo" + +func ownRecords(t *testing.T, f *pubFixture) (rel *records.ReleaseRecord, atts []*records.HealthAttestation) { + t.Helper() + page, err := f.recs.ListOwnSince(context.Background(), netid.Open().String(), 0, 50) + if err != nil { + t.Fatal(err) + } + for _, row := range page.Rows { + env, err := records.Peek(row.Payload) + if err != nil { + t.Fatal(err) + } + if env.Kind == records.KindRelease { + if rel, err = records.DecodeReleaseRecord(row.Payload); err != nil { + t.Fatal(err) + } + } else { + a, err := records.DecodeHealthAttestation(row.Payload) + if err != nil { + t.Fatal(err) + } + atts = append(atts, a) + } + } + return rel, atts +} + +// An old-format import (no NZB store, so a library-derived release ID) is +// published like any other: a release record and an attestation that say the +// ID is files-derived, with size and file count from the grouped files and a +// title taken from the source NZB name when the import history is gone. +func TestOldFormatFilePublishesFilesSchemeRecords(t *testing.T) { + f := newPubInstance(t, level(2), nil) + derived := hx("d") + f.seedFile(t, func(s *seed) { + s.release, s.scheme = derived, database.PeernetSchemeFiles + s.nzbName, s.category, s.indexer = "", "", "" // import history is gone + s.sourceNzb = "/data/.nzbs/12/Some.Movie.2020.1080p.mkv.nzb.gz" + s.size = 3 << 30 + }) + f.seedFile(t, func(s *seed) { + s.path, s.file, s.release, s.scheme = derivedPath2, hx("c"), derived, database.PeernetSchemeFiles + s.nzbName, s.category, s.indexer = "", "", "" + s.sourceNzb = "/data/.nzbs/12/Some.Movie.2020.1080p.mkv.nzb.gz" + s.size = 1 << 20 + }) + f.start(t) + f.svc.NotifyHealthy(healthyNotice()) + f.drain(t) + + if pub := f.status(t).Publish; pub.Published["release"] != 1 || pub.Published["attestation"] != 1 || pub.Errors != 0 { + t.Fatalf("publish status: %+v", pub) + } + rel, atts := ownRecords(t, f) + if rel == nil || len(atts) != 1 { + t.Fatalf("records: %v %d", rel, len(atts)) + } + if rel.Scheme() != records.SchemeFiles || atts[0].Scheme() != records.SchemeFiles { + t.Fatalf("schemes %d / %d, want files for both", rel.Scheme(), atts[0].Scheme()) + } + if rel.FileCount != 2 || rel.TotalSize != 3<<30+1<<20 { + t.Fatalf("count %d size %d: must come from the grouped files", rel.FileCount, rel.TotalSize) + } + if rel.Title != "Some.Movie.2020.1080p" { + t.Fatalf("title %q", rel.Title) + } + if hex.EncodeToString(rel.ReleaseID[:]) != derived { + t.Fatalf("release id %x", rel.ReleaseID) + } +} + +// The same file indexed twice (an import kept under two folders) is one file of +// the release, not two. +func TestReleaseInfoCountsADuplicatedFileOnce(t *testing.T) { + f := newPubInstance(t, level(2), nil) + derived := hx("d") + for _, p := range []string{pubPath, "movies/Copy/movie.mkv"} { + f.seedFile(t, func(s *seed) { s.path, s.release, s.scheme = p, derived, database.PeernetSchemeFiles }) + } + f.start(t) + f.svc.NotifyHealthy(healthyNotice()) + f.drain(t) + rel, _ := ownRecords(t, f) + if rel == nil || rel.FileCount != 1 { + t.Fatalf("release: %+v", rel) + } +} diff --git a/internal/peernet/service/publishing_helpers_test.go b/internal/peernet/service/publishing_helpers_test.go index 34ce17c59..106804eb5 100644 --- a/internal/peernet/service/publishing_helpers_test.go +++ b/internal/peernet/service/publishing_helpers_test.go @@ -124,7 +124,7 @@ func (f *pubFixture) seedFile(t *testing.T, mods ...func(*seed)) { } ctx := context.Background() if s.release != "" || s.file != "" { - if err := f.content.Upsert(ctx, database.PeernetContentEntry{FilePath: s.path, FileID: s.file, ReleaseID: s.release}); err != nil { + if err := f.content.Upsert(ctx, database.PeernetContentEntry{FilePath: s.path, FileID: s.file, ReleaseID: s.release, ReleaseScheme: s.scheme}); err != nil { t.Fatal(err) } } @@ -147,6 +147,8 @@ type seed struct { sourceNzb, password string size int64 postDate time.Time + // scheme is the stored release scheme (0 means nzb when a release is set). + scheme int } func nullIfEmpty(s string) any { diff --git a/internal/peernet/service/releaseinfo.go b/internal/peernet/service/releaseinfo.go index 998cb4ea2..efa6ba69b 100644 --- a/internal/peernet/service/releaseinfo.go +++ b/internal/peernet/service/releaseinfo.go @@ -12,6 +12,7 @@ import ( "github.com/kipsilabs/altmount/internal/database" "github.com/kipsilabs/altmount/internal/nzbname" + "github.com/kipsilabs/altmount/internal/peernet/contentid" "github.com/kipsilabs/altmount/internal/peernet/publish" "github.com/kipsilabs/altmount/internal/peernet/records" ) @@ -21,8 +22,11 @@ const ( infoCacheMax = 256 // maxReleaseFiles bounds how many metadata files are read for one release. // A release with more indexed files than this is not described (it would - // otherwise cost an unbounded number of reads for one record). - maxReleaseFiles = 500 + // otherwise cost an unbounded number of reads for one record). The result + // is cached, so a large release is read once per infoTTL. It is the same + // limit as a derived release ID, so every release that has an ID can be + // described. + maxReleaseFiles = contentid.MaxDerivedReleaseFiles // originProbeFiles is how many files of a release are tried when the // first has no import bookkeeping. originProbeFiles = 5 @@ -30,6 +34,14 @@ const ( maxMediaMetadataBytes = 64 << 10 ) +// releaseScheme maps a stored scheme to the one a record carries. +func releaseScheme(stored int) uint8 { + if stored == database.PeernetSchemeFiles { + return records.SchemeFiles + } + return records.SchemeNZB +} + var errNoReleaseInfo = errors.New("no release information available") // infoSource builds release-level data for a release record. @@ -76,10 +88,17 @@ func (s *infoSource) forRelease(ctx context.Context, releaseHex, hintPath string } var info publish.ReleaseInfo + info.ReleaseScheme = releaseScheme(entries[0].ReleaseScheme) var nzbName, namePassword string secrets := map[string]struct{}{} paths := make([]string, 0, len(entries)) + seenFiles := make(map[string]struct{}, len(entries)) for _, e := range entries { + // The same file kept under two folders is one file of the release. + if _, dup := seenFiles[e.FileID]; dup { + continue + } + seenFiles[e.FileID] = struct{}{} fm, err := s.meta.ReadReleaseFile(e.FilePath) if err != nil || fm == nil { // All or nothing: a partial listing would publish an understated diff --git a/internal/peernet/service/service.go b/internal/peernet/service/service.go index 8159998f9..64a2e1815 100644 --- a/internal/peernet/service/service.go +++ b/internal/peernet/service/service.go @@ -10,6 +10,7 @@ import ( "github.com/kipsilabs/altmount/internal/config" "github.com/kipsilabs/altmount/internal/database" "github.com/kipsilabs/altmount/internal/metadata" + "github.com/kipsilabs/altmount/internal/peernet/bundlex" "github.com/kipsilabs/altmount/internal/peernet/contentid" "github.com/kipsilabs/altmount/internal/peernet/publish" ) @@ -52,6 +53,23 @@ type Deps struct { // Health is the read-only local health lookup the periodic re-announce // requires; nil means release records are never re-announced. Health LocalHealth + // BundleSource, when set, is the full-metadata reader the bundle builder + // uses (tests). When nil it is taken from Metadata if that was made by + // NewMetadataHost. Without one, level 3 announces no bundles. + BundleSource bundlex.BuildSource + // Patches says whether a local PAR2 patch exists for an article. A release + // is shared as a bundle only when it is known that no patch covers it, so + // nil (patch status unknown) shares nothing. Pass bundlex.NoPatches{} for a + // node without a patch store. + Patches bundlex.PatchIndex + // PeerOrigin says whether a library file came from a peer bundle and whether + // its full local check passed. A file that came from a peer bundle is never + // published (record, attestation or bundle) until it has. Nil means no file + // came from a peer bundle. An error means "not eligible". + PeerOrigin PeerBundleOrigin + // BundleImportStatus reports the bundle importer's counters for the + // status; nil means none. + BundleImportStatus func() BundleImportStatus // Metadata is observed (while enabled) to keep the content index current. // May be nil. Metadata MetadataHost @@ -74,20 +92,30 @@ type Deps struct { hub *hintsHub // Test-only hooks. - publishQueueSize int // publishing queue bound (default 512) - publishNow func() time.Time // scheduling clock of publishing (tests; default time.Now) - beforePublish func() // runs before each queued notice is handled - backboneEvery time.Duration // backbone refresh check period (default 10 min) - noContentIndex bool // leave the content index to the test - allowLoopback bool - listenHost string - listenRetryBase time.Duration // first listener rebind delay (default 30 s) - applyHook func(Settings) // runs at the start of every Apply + bundleServeLimits bundlex.ServeLimits // zero: defaults + onWithdraw func([]string) // told which release IDs were withdrawn + wrapProvider func(bundlex.Provider) bundlex.Provider + publishQueueSize int // publishing queue bound (default 512) + publishNow func() time.Time // scheduling clock of publishing (tests; default time.Now) + beforePublish func() // runs before each queued notice is handled + backboneEvery time.Duration // backbone refresh check period (default 10 min) + noContentIndex bool // leave the content index to the test + deriveDebounce time.Duration // quiet period before an incremental derive (default 3 s) + allowLoopback bool + listenHost string + listenRetryBase time.Duration // first listener rebind delay (default 30 s) + applyHook func(Settings) // runs at the start of every Apply // beforeListen runs after the routes are registered and before the // listener starts accepting. beforeListen func(routesRegistered, bound bool) } +// PeerBundleOrigin is the narrow view of the bundle importer the publisher +// needs (bundleimport.Importer.PeerBundleStatus has this shape). +type PeerBundleOrigin interface { + PeerBundleStatus(ctx context.Context, filePath string) (fromPeer, fullCheckPassed bool, err error) +} + // MetadataHost is what the content index needs from the metadata service: // read access to the tree and a place to hook a file observer. type MetadataHost interface { @@ -102,6 +130,10 @@ type metadataHost struct { func (h metadataHost) SetFileObserver(o metadata.FileObserver) { h.ms.SetFileObserver(o) } +func (h metadataHost) bundleBuildSource() bundlex.BuildSource { + return bundlex.NewMetadataSource(h.ms, nil) +} + // NewMetadataHost adapts the metadata service to MetadataHost. func NewMetadataHost(ms *metadata.MetadataService) MetadataHost { return metadataHost{ServiceSource: contentid.NewServiceSource(ms), ms: ms} @@ -135,7 +167,9 @@ func New(deps Deps) *Service { if deps.hub == nil { deps.hub = newHintsHub() } - return &Service{deps: deps, settings: deps.Settings} + s := &Service{deps: deps, settings: deps.Settings} + s.deps.onWithdraw = s.invalidateBundles + return s } // Start brings the feature up when it is enabled. diff --git a/internal/peernet/service/status.go b/internal/peernet/service/status.go index 5a7f84938..cc9a902ab 100644 --- a/internal/peernet/service/status.go +++ b/internal/peernet/service/status.go @@ -72,6 +72,8 @@ type Status struct { Publish PublishStatus // Advice counts what the consuming side did since start. Advice AdviceStatus + // Bundles counts bundle building, serving and fetching since start. + Bundles BundleStatus } // Status returns the current snapshot. @@ -116,6 +118,7 @@ func (s *Service) Status(ctx context.Context) (Status, error) { st.Publish = n.pipe.stats.snapshot() } st.Advice = s.adviceStatus() + st.Bundles = n.bundleStatus(ctx) st.AdvertiseWarning = advertiseWarning(set.AdvertiseAddress, rows, set, n.deps.allowLoopback) for i := range rows { ns, err := n.networkStatus(ctx, &rows[i], set) diff --git a/internal/testsupport/fakepool/fakepool.go b/internal/testsupport/fakepool/fakepool.go index 2cb503552..6acf9d76d 100644 --- a/internal/testsupport/fakepool/fakepool.go +++ b/internal/testsupport/fakepool/fakepool.go @@ -334,6 +334,9 @@ func (c *Client) enter() func() { // behaviorFor resolves the SegmentBehavior for a given message-ID, falling // back to the default. +// BehaviorFor returns the behavior currently configured for a message-ID. +func (c *Client) BehaviorFor(messageID string) SegmentBehavior { return c.behaviorFor(messageID) } + func (c *Client) behaviorFor(messageID string) SegmentBehavior { c.mu.RLock() defer c.mu.RUnlock() From ec56dc213028f96db91624ca60f0db3f2275557b Mon Sep 17 00:00:00 2001 From: javi11 Date: Sat, 14 Nov 2026 15:25:43 +0100 Subject: [PATCH 2/5] fix(peernet): bind bundle layouts and wire header verification Second hardening round for bundle imports, from a re-review and a live two-instance run. - wire the article header reader into the bundle importer; without it every peer bundle import was rejected. The importer now refuses to be built with a required collaborator missing - a pool entry may appear once per table, tables may share entries only when identical, and only the first and last segment of a table may be partial - every table is sampled (first, last and a random article); bundles with too many tables to sample are rejected; the forced full check also verifies headers on a sample of articles - the open network accepts only plain layouts in containers with a structural check; archive, keyed and clipped layouts stay private-only - stricter fragmented MP4 tail, interior checks for Matroska and transport streams - corroboration counts the address actually connected to, so several names for one server count once - failure paths compare the file ID before removing anything - a degraded or failed fast-path import falls back to a normal import - offers are filtered before dialling; fetch limits and plain refusals are reported as such - lower decode memory, one verification at a time, 16 MiB decoded cap - catch-up publish when a network level is raised --- cmd/altmount/cmd/peernet_bundle_wiring.go | 66 ++++-- .../cmd/peernet_bundle_wiring_test.go | 29 +++ cmd/altmount/cmd/peernet_stremio_wiring.go | 7 + cmd/altmount/cmd/setup.go | 2 + docs/docs/3. Configuration/peer-network.md | 10 +- .../specs/2026-11-12-peer-network-design.md | 23 +- .../config/peernet/ActivityCard.tsx | 2 +- .../components/config/peernet/peernetStats.ts | 26 ++- frontend/tests/peernetStats.test.ts | 7 + internal/api/sabnzbd_peer_hold_delete_test.go | 75 ++++++ internal/api/stremio_peer.go | 21 ++ internal/api/stremio_peer_test.go | 11 + ...eernet_connected_ip_and_failed_bundles.sql | 28 +++ ...eernet_connected_ip_and_failed_bundles.sql | 28 +++ .../peernet_bundle_import_repository.go | 114 ++++++++++ .../peernet_bundle_import_repository_test.go | 26 +++ internal/database/peernet_bundle_queries.go | 24 ++ internal/database/peernet_peer_repository.go | 53 ++++- .../database/peernet_peer_repository_test.go | 31 +++ internal/database/peernet_queue_hold_test.go | 40 ++++ .../peernet_sync_cursor_repository.go | 19 ++ internal/database/repository.go | 20 ++ internal/health/checker.go | 14 +- internal/health/peer_bundle.go | 29 ++- internal/health/peer_headers.go | 88 ++++++++ internal/health/peer_verdicts_test.go | 110 +++++++++ internal/health/worker.go | 14 ++ .../filesystem/safefoldername_test.go | 19 ++ internal/importer/filesystem/utils.go | 18 ++ internal/importer/peer_bundle.go | 19 ++ .../importer/peer_bundle_prebuilt_test.go | 42 ++++ internal/importer/peer_bundle_service.go | 27 +++ internal/importer/processor.go | 5 +- internal/peernet/bridge/bundleimport.go | 64 +++++- internal/peernet/bridge/bundleimport_test.go | 33 ++- internal/peernet/bundle/budget_test.go | 4 +- internal/peernet/bundle/compress.go | 25 +- internal/peernet/bundle/compress_test.go | 2 +- internal/peernet/bundle/fuzz_test.go | 7 +- internal/peernet/bundle/heap_test.go | 92 ++++++++ .../peernet/bundle/layout_binding_test.go | 102 +++++++++ internal/peernet/bundle/limits.go | 10 +- internal/peernet/bundle/race_off_test.go | 5 + internal/peernet/bundle/race_on_test.go | 6 + internal/peernet/bundle/validate.go | 79 ++++++- internal/peernet/bundle/validate_test.go | 2 +- internal/peernet/bundle/verify.go | 44 ++-- internal/peernet/bundle/verify_test.go | 80 ++++--- internal/peernet/bundle/wire.go | 58 ++++- internal/peernet/bundle/wire_decode.go | 5 +- internal/peernet/bundleimport/accept.go | 63 +++++- internal/peernet/bundleimport/accept_test.go | 36 ++- .../peernet/bundleimport/foreignfile_test.go | 110 +++++++++ internal/peernet/bundleimport/headers_test.go | 129 +++++++++++ internal/peernet/bundleimport/holds_test.go | 99 ++++++++ internal/peernet/bundleimport/importer.go | 32 +++ internal/peernet/bundleimport/pathobserver.go | 11 +- internal/peernet/bundleimport/reconcile.go | 3 + internal/peernet/bundleimport/sample.go | 213 +++++++++++++----- internal/peernet/bundleimport/structure.go | 201 +++++++++++++++-- .../bundleimport/structure_forgery_test.go | 170 ++++++++++++++ .../bundleimport/structure_variants_test.go | 7 +- internal/peernet/bundleimport/tracker.go | 119 +++++++++- internal/peernet/bundleimport/tracker_test.go | 7 +- internal/peernet/bundleimport/types.go | 107 +++++++-- .../peernet/bundleimport/verify_headers.go | 135 +++++++++++ internal/peernet/bundleimport/yenc_test.go | 163 ++++++++++++-- internal/peernet/bundlex/buckets.go | 9 + internal/peernet/bundlex/builder.go | 81 ++++++- internal/peernet/bundlex/dos_test.go | 76 +++++++ internal/peernet/bundlex/fetch.go | 155 +++++++++++-- internal/peernet/bundlex/fetch_test.go | 209 ++++++++++++++++- internal/peernet/bundlex/serve.go | 2 +- internal/peernet/discovery/client.go | 3 + internal/peernet/discovery/discovery_test.go | 45 ++++ internal/peernet/discovery/service.go | 14 +- internal/peernet/discovery/status.go | 10 + internal/peernet/discovery/tick.go | 14 ++ internal/peernet/recsync/exchange.go | 3 + internal/peernet/recsync/exchange_test.go | 8 + internal/peernet/recsync/status.go | 15 +- .../peernet/service/bundle_import_e2e_test.go | 31 ++- internal/peernet/service/bundles.go | 47 +++- .../peernet/service/bundles_failed_test.go | 29 +++ internal/peernet/service/catchup_test.go | 74 ++++++ internal/peernet/service/publishing.go | 76 +++++++ internal/peernet/service/service.go | 4 + internal/peernet/transport/client.go | 19 +- 88 files changed, 3887 insertions(+), 307 deletions(-) create mode 100644 internal/api/sabnzbd_peer_hold_delete_test.go create mode 100644 internal/database/migrations/postgres/063_add_peernet_connected_ip_and_failed_bundles.sql create mode 100644 internal/database/migrations/sqlite/063_add_peernet_connected_ip_and_failed_bundles.sql create mode 100644 internal/health/peer_headers.go create mode 100644 internal/health/peer_verdicts_test.go create mode 100644 internal/importer/filesystem/safefoldername_test.go create mode 100644 internal/importer/peer_bundle_prebuilt_test.go create mode 100644 internal/peernet/bundle/heap_test.go create mode 100644 internal/peernet/bundle/layout_binding_test.go create mode 100644 internal/peernet/bundle/race_off_test.go create mode 100644 internal/peernet/bundle/race_on_test.go create mode 100644 internal/peernet/bundleimport/foreignfile_test.go create mode 100644 internal/peernet/bundleimport/headers_test.go create mode 100644 internal/peernet/bundleimport/holds_test.go create mode 100644 internal/peernet/bundleimport/structure_forgery_test.go create mode 100644 internal/peernet/bundleimport/verify_headers.go create mode 100644 internal/peernet/service/bundles_failed_test.go create mode 100644 internal/peernet/service/catchup_test.go diff --git a/cmd/altmount/cmd/peernet_bundle_wiring.go b/cmd/altmount/cmd/peernet_bundle_wiring.go index 1decb942e..6d1c5be7a 100644 --- a/cmd/altmount/cmd/peernet_bundle_wiring.go +++ b/cmd/altmount/cmd/peernet_bundle_wiring.go @@ -70,6 +70,31 @@ func (s *peernetBundleSeams) FetchBundle(ctx context.Context, releaseID string) return f.FetchBundle(ctx, releaseID) } +// FetchBundleWith forwards the caller's requirements to the connected fetcher +// when it can apply them before dialing; otherwise it fetches unfiltered. +func (s *peernetBundleSeams) FetchBundleWith(ctx context.Context, releaseID string, o bundleimport.FetchOptions) (*bundle.Bundle, bundleimport.Origin, error) { + s.mu.RLock() + f := s.fetcher + s.mu.RUnlock() + if f == nil { + return nil, bundleimport.Origin{}, errNoBundleFetcher + } + if of, ok := f.(bundleimport.OptionFetcher); ok { + return of.FetchBundleWith(ctx, releaseID, o) + } + return f.FetchBundle(ctx, releaseID) +} + +// ReportFailedBundle forwards a failed layout to the connected reporter. +func (s *peernetBundleSeams) ReportFailedBundle(ctx context.Context, bundleID, releaseID string) { + s.mu.RLock() + r := s.reporter + s.mu.RUnlock() + if fr, ok := r.(bundleimport.BundleFailedReporter); ok { + fr.ReportFailedBundle(ctx, bundleID, releaseID) + } +} + // ReportBadBundle forwards to the connected reporter; with none it logs. func (s *peernetBundleSeams) ReportBadBundle(ctx context.Context, o bundleimport.Origin, releaseID string) { s.mu.RLock() @@ -266,24 +291,37 @@ func newPeernetBundleHost(d bundleWiringDeps, seams *peernetBundleSeams) *peerne return err != nil || n > 0 }, build: func() *bundleimport.Importer { - return bundleimport.New(bundleimport.Config{ - Library: d.Metadata, - Stager: bundleimport.NewFSStager(bundleimport.FSStagerDeps{Pool: d.Pool, Config: d.GetConfig, Cache: d.Cache, Root: stagingRoot}), - Prober: bridge.NewReleaseProber(d.Importer), - State: state, - Index: database.NewPeernetContentRepository(conn, dialect), - Fetcher: seams, - Recorder: bridge.NewRecorder(d.Importer), - Announcer: bridge.NewAnnouncer(d.Importer), - Reporter: seams, - HealthEnabled: func() bool { return d.GetConfig().GetHealthEnabled() }, - WillBeChecked: func(p string) bool { return postprocessor.WillScheduleHealth(d.GetConfig(), p) }, - AllowedExtensions: func() []string { return d.GetConfig().Import.AllowedFileExtensions }, - }) + cfg := bundleImporterConfig(d, state, stagingRoot, seams) + if missing := cfg.Missing(); len(missing) > 0 { + // A programming error: the accept pipeline would reject every + // bundle at runtime. Say so loudly instead of failing quietly. + slog.Error("Peer bundle import is wired without required collaborators", "missing", missing) + } + return bundleimport.New(cfg) }, } } +// bundleImporterConfig is the production configuration of the bundle importer. +func bundleImporterConfig(d bundleWiringDeps, state *database.PeernetBundleImportRepository, stagingRoot string, seams *peernetBundleSeams) bundleimport.Config { + conn, dialect := d.DB.Connection(), d.DB.Dialect() + return bundleimport.Config{ + Library: d.Metadata, + Stager: bundleimport.NewFSStager(bundleimport.FSStagerDeps{Pool: d.Pool, Config: d.GetConfig, Cache: d.Cache, Root: stagingRoot}), + Prober: bridge.NewReleaseProber(d.Importer), + State: state, + Index: database.NewPeernetContentRepository(conn, dialect), + Fetcher: seams, + Headers: bundleimport.NewPoolHeaderReader(d.Pool), + Recorder: bridge.NewRecorder(d.Importer), + Announcer: bridge.NewAnnouncer(d.Importer), + Reporter: seams, + HealthEnabled: func() bool { return d.GetConfig().GetHealthEnabled() }, + WillBeChecked: func(p string) bool { return postprocessor.WillScheduleHealth(d.GetConfig(), p) }, + AllowedExtensions: func() []string { return d.GetConfig().Import.AllowedFileExtensions }, + } +} + // connectPeernetBundleSeams connects the peer network service as the bundle // fetcher and the bad-publisher reporter, then re-applies the configuration so // the importer fast path appears. With the feature off nothing is installed. diff --git a/cmd/altmount/cmd/peernet_bundle_wiring_test.go b/cmd/altmount/cmd/peernet_bundle_wiring_test.go index 637a029dc..5f26d18b9 100644 --- a/cmd/altmount/cmd/peernet_bundle_wiring_test.go +++ b/cmd/altmount/cmd/peernet_bundle_wiring_test.go @@ -10,6 +10,7 @@ import ( "github.com/kipsilabs/altmount/internal/database" "github.com/kipsilabs/altmount/internal/health" "github.com/kipsilabs/altmount/internal/importer" + "github.com/kipsilabs/altmount/internal/metadata" "github.com/kipsilabs/altmount/internal/peernet/bundle" "github.com/kipsilabs/altmount/internal/peernet/bundleimport" "github.com/kipsilabs/altmount/internal/peernet/service" @@ -265,3 +266,31 @@ func TestPeernetStagingRoot_IsAlwaysAbsoluteAndNeverTheWorkingDirectory(t *testi } assert.Equal(t, "/var/lib/altmount/.peernet-staging", peernetStagingRoot("/var/lib/altmount/altmount.db")) } + +// The production importer configuration must carry every collaborator the +// accept pipeline needs: a nil Headers once made every bundle fail with +// "no way to read article headers" at runtime while all unit tests passed. +func TestProductionBundleImporterConfigHasEveryRequiredCollaborator(t *testing.T) { + db, err := database.NewDB(database.Config{DatabasePath: filepath.Join(t.TempDir(), "db")}) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + cfg := config.DefaultConfig() + d := bundleWiringDeps{ + Metadata: metadata.NewMetadataService(t.TempDir()), DB: db, Importer: &importer.Service{}, + GetConfig: func() *config.Config { return cfg }, + } + state := database.NewPeernetBundleImportRepository(db.Connection(), db.Dialect()) + ic := bundleImporterConfig(d, state, t.TempDir(), &peernetBundleSeams{}) + assert.Empty(t, ic.Missing(), "a missing collaborator makes the feature silently never work") + assert.NotNil(t, ic.Headers, "without a header reader every bundle is rejected as probe_inconclusive") + + var bad bundleimport.Config + assert.Contains(t, bad.Missing(), "Headers", "the check itself must notice a nil Headers") +} + +// The forced full check's header sample reaches the health worker through the +// gate: the importer must satisfy both interfaces. +var ( + _ health.PeerBundleGate = (*bundleimport.Importer)(nil) + _ health.PeerHeaderVerifier = (*bundleimport.Importer)(nil) +) diff --git a/cmd/altmount/cmd/peernet_stremio_wiring.go b/cmd/altmount/cmd/peernet_stremio_wiring.go index 006ab18ca..eab0f0c4f 100644 --- a/cmd/altmount/cmd/peernet_stremio_wiring.go +++ b/cmd/altmount/cmd/peernet_stremio_wiring.go @@ -6,6 +6,7 @@ import ( "github.com/kipsilabs/altmount/internal/api" "github.com/kipsilabs/altmount/internal/peernet/bundleimport" + "github.com/kipsilabs/altmount/internal/peernet/bundlex" "github.com/kipsilabs/altmount/internal/peernet/service" ) @@ -67,6 +68,12 @@ func (p stremioPeerPlayback) Play(ctx context.Context, req api.PeerPlayRequest) DirFunc: func(title string) string { return p.dirs.PeerImportDir(req.Category, title) }, }) if err != nil { + switch { + case errors.Is(err, bundlex.ErrRateLimited): + return api.PeerPlayResult{}, &api.PeerRetryError{After: 60, Err: err} + case errors.Is(err, bundlex.ErrBackedOff): + return api.PeerPlayResult{}, &api.PeerRetryError{After: 300, Err: err} + } return api.PeerPlayResult{}, err } return api.PeerPlayResult{Paths: res.Paths, Sizes: res.Sizes}, nil diff --git a/cmd/altmount/cmd/setup.go b/cmd/altmount/cmd/setup.go index 89df350fd..c618f3f06 100644 --- a/cmd/altmount/cmd/setup.go +++ b/cmd/altmount/cmd/setup.go @@ -505,6 +505,8 @@ func peernetDeps( // Record exchange: stored records and the per-peer sync positions. Records: database.NewPeernetRecordRepository(conn, d), Cursors: database.NewPeernetSyncCursorRepository(conn, d), + // Negative cache of Bundle IDs that failed local verification. + BundleState: database.NewPeernetBundleImportRepository(conn, d), // Publishing of this node's own healthy releases (levels 2 and 3). PublishState: database.NewPeernetPublishStateRepository(conn, d), Origins: database.NewPeernetOriginRepository(conn, d), diff --git a/docs/docs/3. Configuration/peer-network.md b/docs/docs/3. Configuration/peer-network.md index 912cb58fa..a8560757b 100644 --- a/docs/docs/3. Configuration/peer-network.md +++ b/docs/docs/3. Configuration/peer-network.md @@ -181,7 +181,13 @@ Before anything is written the layout is also checked against the articles thems - **Folder name.** The release folder comes from the title with the media extension removed first; names that are empty, hidden (start with a dot), reserved by AltMount or equal to a category folder are refused, so a title can never point outside the category folder. - **Offers.** Of several offers, the bundle announced identically by publishers in the most distinct network ranges is preferred; a publisher's own attestation never raises the rank of its own offer. -What this proves: the articles exist on your providers, sampled articles match the layout, and the media container is structurally sound. It does not prove the whole file is the titled content; that is the same risk as a mislabelled NZB from an indexer. It does not prove the whole file is the titled content; that is the same risk as a mislabelled NZB from an indexer. +- **Plain layouts only from the open network.** A bundle offered through the open network is accepted only when every file is exactly one plain run of articles: archive volumes (nested layouts), encryption keys and multi-clip files are refused with `layout_not_allowed_open`, and media whose container has no structural check (anything but Matroska/WebM, MP4/MOV and MPEG-TS/M2TS) with `container_not_verifiable`. This is a stated limitation, not a bug: those layouts cannot be bound to the articles without reading archive headers, which is planned. Private networks still accept them. +- **Sampling.** Every run of articles is sampled: its first and last article and at least one random one in between (one per 500 for a media file, at least three) have their yEnc headers (part size, offset, total) compared with the layout, up to 600 articles and 512 MiB; a release with more archive volumes than that is refused (`too_many_tables`). A single movie file reads about 5 articles (4 MB), a 50-volume release 150 (115 MB), a 200-volume release 600 (461 MB). With your NZB at hand the first article's yEnc file name must also match the NZB entry it belongs to. The forced full check reads a further random sample of headers (1 percent of the articles, 20 to 500) and fails the import on a contradiction. It samples; it does not prove. +- **Media structure.** Besides the end of the file, the container is read at random positions inside the file (Matroska: three random 1 MiB windows must contain cluster data; MPEG-TS: eight random packet positions must be in sync; fragmented MP4 must end in a real `moof`+`mdat` pair or `mfra`). What remains unprovable: bytes inside MP4 `mdat`, inside a Matroska frame or between sampled positions. +- **Degraded means failed.** A peer release whose full check finds the media playable but with missing segments is failed and removed, never announced. When you supplied the NZB the queue item is not marked failed: it is imported normally from your NZB, once, so you get what a normal import would have produced. A release imported directly (Stremio peer play) is failed. A missing article alone does not count against the publisher (it can be propagation just after posting); only a contradiction between the layout and the articles' own headers does, and that layout is then ignored whoever offers it, for 7 days. +- **Corroboration** counts distinct /24 (IPv4) or /48 (IPv6) ranges of the address this node actually connected to when it last verified the publisher, never host names; with the open-network switch off, open-network publishers are not even contacted. + +What this proves: the articles exist on your providers, sampled articles match the layout, and the media container is structurally sound at the places it was read. It does not prove that the content between the sampled points is the titled media; that is the same class of risk as a mislabelled NZB from an indexer. ### Stremio peer candidates @@ -191,7 +197,7 @@ Playing one asks your node (not the player) to fetch the bundle, verify it as de ### Limits -- Bundles: 8 MB compressed, 64 MB decompressed, 500,000 segments and 10,000 files at most; a receiving node also refuses releases over 2 TiB or 2,000 files. +- Bundles: 8 MB compressed, 16 MB decompressed (about 340,000 segments of a typical release, roughly 250 GB), 500,000 segments and 10,000 files at most; a receiving node also refuses releases over 2 TiB or 2,000 files. - Fetching: on demand only, never speculative; at most 3 attempts of 60 seconds per request, 60 attempts per hour overall, 12 per publisher, 2 downloads at once. - The health check must be enabled to receive bundles, because the full check must be able to run. - A bundle is for the exact release: two NZBs of the same movie with different article sets are different releases. diff --git a/docs/superpowers/specs/2026-11-12-peer-network-design.md b/docs/superpowers/specs/2026-11-12-peer-network-design.md index b0df0bda0..2554091c2 100644 --- a/docs/superpowers/specs/2026-11-12-peer-network-design.md +++ b/docs/superpowers/specs/2026-11-12-peer-network-design.md @@ -222,7 +222,7 @@ These are the core of the security model. - No peer input is parsed as XML, used in a template, used to build a command, or interpolated into SQL. NZB XML is only generated outward, with escaping. - The UI renders peer strings as text only. -- **Hard limits**: record 2 KB; bundle 8 MB compressed and 64 MB decompressed, +- **Hard limits**: record 2 KB; bundle 8 MB compressed and 16 MB decompressed (lowered from 64 MB in the second hardening round), enforced while streaming; 500k segments and 10k files per bundle; per-peer request rates; a storage cap for the index with oldest-first eviction; a quota per publisher; records expire after 30 days unless re-announced. (Numbers are @@ -542,7 +542,7 @@ Release ID of the other scheme, or with a trimmed segment list, fails. ### Limits (exported constants) -8 MB compressed, 64 MB decompressed (enforced while streaming, 8 MB zstd +8 MB compressed, 16 MB decompressed (was 64 MB) (enforced while streaming, 8 MB zstd window, single-threaded decoder, no goroutines left behind), 500,000 segments (pool, and positions over all tables), 10,000 files, 50,000 NZB file entries, message-ID 248 bytes, file name 255 bytes, title 256 bytes, password 256 bytes, @@ -977,3 +977,22 @@ A security and a correctness review of the receiving half of phase 3 produced th - **Stremio listing.** Peer candidates are looked up concurrently with the indexer search and collected afterwards with at most a 250 ms grace; a slow, failing or panicking lookup yields no peer streams. The merge (library, peer, indexer entries) is one function and byte-identical to the former output when there are no peer streams. - **Cheap no-offer import.** The importer asks `MayHaveOffers` first: a cached answer, recomputed only when a bundle-announcing record was stored (`BundleOfferGeneration`), so with no offer nothing is built or hashed. When offers exist the store built for the lookup is reused by the import's parse unless gap placeholders changed the NZB. - **Structure check variants.** See the matrix in the user documentation: SeekHead-less and unknown-size Matroska, fragmented MP4, trailing boxes and zero padding, 64-bit and zero-sized boxes, and 188, 192 and 204 byte transport streams are accepted; a layout correct only at the head is still rejected, because the end of the file is always read (SeekHead target, box chain, fragment box ending at EOF, sync on the last packet). The weakest case is a SeekHead-less Matroska, where the end is checked by the Segment size and readable tail only. + +## Phase 3: second hardening round + +A second review with working PoCs found that the identities bind less than the first round assumed: the File ID and the Release ID hash only message-IDs, so sizes, usable ranges, extents, keys and names are bound by nothing but local checks, and the forced full check is STAT-only, so a size or offset lie that survives acceptance is permanent. The rules below close those gaps as far as sampling can. + +**Layout binding.** + +- A pool entry is referenced at most once within a segment table, and by one table only: two tables may share an entry only when they are the identical table (every reference, size and usable range equal, as for a shared outer volume or a PAR2 file listed by several files). Tables are keyed everywhere (validation, sampling) by `bundle.KeyOf`, a hash of the whole table, never by their first message-id. PoCs P1 (one article referenced 1,999 times as a 1.4 GB file) and P4 (two files sharing a first article, the second never sampled) are regression tests. +- Interior segments of every table are used whole (`Start == 0`, `End == Size-1`); only the first may start inside its article and only the last may end inside it (`ErrPartialRange`). Real archive slicing in this importer is expressed by extents into outer volumes, never by interior partial ranges of an inline table, so no other layout kind was needed. PoC P2 (100 articles of 700 KB each used for 10 bytes) is a regression test. In the files scheme a declared segment size is bounded by 4 MiB (`MaxFilesSchemeSegment`): posters use 0.25 to 1 MB articles (the importer's own estimates assume about 768 KB) and nothing in the importer or the readers expects more; the nzb scheme keeps the 8 MiB wire cap and is additionally bounded by the user's NZB. +- Sampling. Every table gets its first and last article and at least one uniformly random interior article (crypto/rand); tables of media files get one per 500 segments, at least three. The cap is 600 articles and 512 MiB (`MaxYencSamples`, `MaxYencBytes`) within 90 seconds (`YencTimeout`). A layout whose tables do not fit at the minimum rate is rejected (`too_many_tables`), never left unsampled. Measured at 768 KB articles: a single 1.1 GB file reads 5 articles (4 MB); a 50-volume release 150 articles (115 MB); a 200-volume release 600 articles (461 MB); 201 volumes are rejected. When the user's NZB is present the first article's yEnc `name=` must equal the subject file name of the NZB entry the table reads (obfuscated names bind too); a disagreement with the local NZB rejects the bundle but is not reported against the publisher. +- Forced full check. A pending peer import additionally gets a header sample proportional to its size (1 percent of its articles, at least 20, at most 500, bounded bytes) through an optional `health.PeerHeaderVerifier` installed together with the peer gate (nil: no change; `internal/health` imports nothing from the peer network). A mismatch fails the import and is the only failure attributable to the publisher. This is sampling, not proof. +- Open network and nested layouts. Archive layouts (outer volumes, extents, AES keys, clips) cannot be bound without reading archive headers. In the open network only plain layouts are accepted (`layout_not_allowed_open`); private networks keep nested layouts, and what a node may serve is unchanged. Open-network bundles for archive releases need archive-header spot checks (planned). +- Structure check. A fragmented MP4 must end in a `moof`+`mdat` pair whose sizes chain to the end of the file or an `mfra` with a valid `mfro` (optionally followed by `free`/`skip`); a non-fragmented MP4 walks up to 256 top-level box headers instead of a 64-read cap; Matroska reads three random interior positions of 1 MiB (three tries each) and requires two consecutive Cluster-interior elements (SimpleBlock, BlockGroup, Timecode, Cluster) in each; MPEG-TS checks eight random positions on the packet grid. The open network accepts only containers with a real check (Matroska/WebM, MP4/MOV, MPEG-TS/M2TS); other media is refused there (`container_not_verifiable`). Still unprovable: bytes inside `mdat`, inside a Matroska frame or between sampled TS packets, and everything between sampled positions. A forger who puts real structure everywhere the check can look, and garbage elsewhere, is not detected by structure alone. + +**Trust and fairness.** Corroboration (bundle offers and phase 2 hints, `KnownPrefixes`) keys on the remote IP actually connected to when the peer was last verified (`peernet_peers.connected_ip`, migration 063; /24 for IPv4, /48 for IPv6); a peer with none counts for nothing, so two DNS names for one server are one range. Attesters that rank offers must themselves be known. The fast path hands its requirements (open allowed, corroboration floor) to the fetcher, which filters offers before dialing anyone, so with the switch off no open-network peer is contacted. The serve-side build budget is spent only after the build slot is held and the build starts, and a requester (node ID, public address) can use at most a sixth of it with a burst of one; cache hits and refused requests cost nothing. Only one bundle is verified at a time node-wide (`bundle.VerifyContext`); `MaxDecompressedSize` is 16 MiB (a 50 GB release is about 3.2 MB canonical, so roughly 340,000 segments, about 250 GB at 768 KB per article, still fits comfortably). Decode peak heap fell from 6.5 times the canonical size to about 3.3 times (single exact read buffer, streaming canonical comparison instead of a second full re-encode). + +**State and failure.** Every action on a row's path (failure, hold, forced check, stale escalation) first compares the File ID at the path with the row; a foreign row is dropped and causes no delete, hold or forced check. A delete followed by a rewrite of the same File ID at the same path ends the peer import cleanly instead of re-attaching to a normal import. A degraded verdict fails a pending peer import at once (never announced, removed, no PAR2 repair queued); on the fast path the queue item is put back for a normal import with the peer fast path skipped once (`RequeueNormalImport`), a direct import is failed. A check that failed outright and ended corrupted counts like a corrupted verdict. The publisher is reported only for a header contradiction, not for missing articles (which can be propagation just after posting), and then the layout goes on a negative cache keyed by Bundle ID (`peernet_failed_bundles`, 7 days, persisted because failures surface up to a week later) that the fetcher consults before dialing. Housekeeping releases holds of finished imports owing no ARR (`SettledHolds`) in addition to orphaned ones, and `Reconcile` takes the write lock exclusively so it can never roll back a write in flight. Deleting a held queue item by download ID or path removes its provisional history. Idempotent `SafeFolderName`, the dead `Options.FilesReleaseID` hook, and the prebuilt NZB store reused after `TrimArticles` were fixed. + +**Residual risk.** Content between sampled points is not proven to be the titled media. That is the same class of risk as a mislabelled NZB from an indexer, now bounded by random sampling per table, header samples during the full check, structural checks at random interior positions, plain layouts only in the open network, and a fallback to a normal import when a peer layout does not hold up. diff --git a/frontend/src/components/config/peernet/ActivityCard.tsx b/frontend/src/components/config/peernet/ActivityCard.tsx index 7f9362154..cee4dbb5c 100644 --- a/frontend/src/components/config/peernet/ActivityCard.tsx +++ b/frontend/src/components/config/peernet/ActivityCard.tsx @@ -94,7 +94,7 @@ function BreakdownList({ title, counts, label }: BreakdownListProps) { function BundlesBlock({ status }: { status: PeernetStatus }) { const b = status.bundles; const imp = b?.import; - const serving = showBundleServing(status.networks); + const serving = showBundleServing(status.networks, b); const backedOff = count(b?.fetch_backed_off); return (
diff --git a/frontend/src/components/config/peernet/peernetStats.ts b/frontend/src/components/config/peernet/peernetStats.ts index cf4bf3093..59ae25e06 100644 --- a/frontend/src/components/config/peernet/peernetStats.ts +++ b/frontend/src/components/config/peernet/peernetStats.ts @@ -79,7 +79,7 @@ const SERVE_REFUSED_LABELS: ReadonlyMap = new Map([ ]); const FETCH_FAILED_LABELS: ReadonlyMap = new Map([ - ["refused", "Peer refused the request"], + ["refused", "The peer declined to serve this bundle"], ["bad_status", "Peer answered with an error"], ["transport", "Connection problem"], ["truncated", "Download was cut short"], @@ -113,6 +113,15 @@ const IMPORT_REJECT_LABELS: ReadonlyMap = new Map([ ["size_implausible", "Declared file sizes do not fit your NZB"], ["yenc_mismatch", "Sampled articles contradict the bundle"], ["structure_invalid", "Media file structure is broken"], + [ + "layout_not_allowed_open", + "Archive or encrypted layouts are not accepted from the open network", + ], + [ + "container_not_verifiable", + "Media container cannot be checked, so the open network does not carry it", + ], + ["too_many_tables", "More archive volumes than the verification budget covers"], ]); export function notShareableLabel(key: string): string { @@ -214,9 +223,18 @@ export function showBundlesBlock(networks: readonly HasLevel[]): boolean { return networks.some((n) => n.level >= 1); } -/** Only level 3 serves bundles. */ -export function showBundleServing(networks: readonly HasLevel[]): boolean { - return networks.some((n) => n.level >= 3); +/** + * Level 3 serves bundles. The serving numbers also show whenever any of them is + * non-zero (announced, built, served, refused), so they never stay hidden while + * the API reports them, whatever level a network is shown at. + */ +export function showBundleServing(networks: readonly HasLevel[], bundles?: unknown): boolean { + if (networks.some((n) => n.level >= 3)) return true; + if (!bundles || typeof bundles !== "object") return false; + const b = bundles as Record; + return ["announced", "built", "served", "served_bytes", "not_shareable", "serve_refused"].some( + (key) => anyPositive(b[key]), + ); } function positive(n: unknown): boolean { diff --git a/frontend/tests/peernetStats.test.ts b/frontend/tests/peernetStats.test.ts index a9a7e9763..62977e8c2 100644 --- a/frontend/tests/peernetStats.test.ts +++ b/frontend/tests/peernetStats.test.ts @@ -138,6 +138,9 @@ const BACKEND_IMPORT_REJECT_KEYS = [ "size_implausible", "yenc_mismatch", "structure_invalid", + "layout_not_allowed_open", + "container_not_verifiable", + "too_many_tables", ]; describe("bundle reason coverage", () => { @@ -167,6 +170,10 @@ describe("bundle block visibility", () => { expect(showBundlesBlock([{ level: 1 }])).toBe(true); expect(showBundleServing([{ level: 2 }])).toBe(false); expect(showBundleServing([{ level: 1 }, { level: 3 }])).toBe(true); + // Serving numbers the API reports are never hidden by the level check. + expect(showBundleServing([{ level: 1 }], { announced: 8 })).toBe(true); + expect(showBundleServing([{ level: 2 }], { serve_refused: { busy: 2 } })).toBe(true); + expect(showBundleServing([{ level: 2 }], { announced: 0, fetched: 4 })).toBe(false); }); test("activity detection ignores junk", () => { expect(bundleHasActivity(undefined)).toBe(false); diff --git a/internal/api/sabnzbd_peer_hold_delete_test.go b/internal/api/sabnzbd_peer_hold_delete_test.go new file mode 100644 index 000000000..d0574635b --- /dev/null +++ b/internal/api/sabnzbd_peer_hold_delete_test.go @@ -0,0 +1,75 @@ +package api + +import ( + "context" + "database/sql" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/gofiber/fiber/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/importer" +) + +// heldPeerItem is a completed queue item of a pending peer import with the +// provisional history row the import recorded (no download id on the history +// row: only the queue row carries it). +func heldPeerItem(t *testing.T, conn *sql.DB, repo *database.Repository, downloadID string) int64 { + t.Helper() + ctx := context.Background() + item := &database.ImportQueueItem{DownloadID: &downloadID, NzbPath: filepath.Join(t.TempDir(), "x.nzb"), + Status: database.QueueStatusProcessing, Priority: database.QueuePriorityNormal, MaxRetries: 1} + require.NoError(t, repo.AddToQueue(ctx, item)) + nzb := item.ID + require.NoError(t, repo.AddImportHistory(ctx, &database.ImportHistory{NzbID: &nzb, NzbName: "x.nzb", FileName: "a.mkv", FileSize: 10, VirtualPath: "tv/x"})) + require.NoError(t, repo.UpdateQueueItemStatus(ctx, item.ID, database.QueueStatusCompleted, nil)) + require.NoError(t, database.NewPeernetBundleImportRepository(conn, database.DialectSQLite).SetQueueHold(ctx, item.ID, true)) + return item.ID +} + +func historyIDs(t *testing.T, repo *database.Repository) int { + t.Helper() + rows, err := repo.ListSABnzbdHistory(context.Background(), "", 100, 0) + require.NoError(t, err) + return len(rows) +} + +// Deleting the queue row of a held peer import (by download id) must take its +// provisional history with it: the history view would otherwise show a +// Completed job for a release that is still unverified. +func TestPeerHold_QueueDeleteByDownloadIDDropsTheProvisionalHistory(t *testing.T) { + server, repo, conn := newPeerHoldServer(t) + heldPeerItem(t, conn, repo, "dl-held-1") + + app := fiber.New() + app.Delete("/sabnzbd", server.handleSABnzbdQueueDelete) + resp, err := app.Test(httptest.NewRequest("DELETE", "/sabnzbd?value=dl-held-1&name=queue", nil)) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + + assert.Zero(t, historyIDs(t, repo), "no Completed ghost in the history") +} + +func TestPeerHold_DeleteQueueItemsByPathDropsTheProvisionalHistory(t *testing.T) { + _, repo, conn := newPeerHoldServer(t) + id := heldPeerItem(t, conn, repo, "dl-held-2") + item, err := repo.GetQueueItem(context.Background(), id) + require.NoError(t, err) + + require.NoError(t, repo.DeleteQueueItemsByPath(context.Background(), item.NzbPath)) + + assert.Zero(t, historyIDs(t, repo)) +} + +func newPeerHoldServer(t *testing.T) (*Server, *database.Repository, *sql.DB) { + t.Helper() + db, err := database.NewDB(database.Config{Type: "sqlite", DatabasePath: filepath.Join(t.TempDir(), "test.db")}) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + repo := database.NewRepository(db.Connection(), db.Dialect()) + return &Server{queueRepo: repo, importerService: &importer.Service{}}, repo, db.Connection() +} diff --git a/internal/api/stremio_peer.go b/internal/api/stremio_peer.go index 99ccc705f..2b2dc9b04 100644 --- a/internal/api/stremio_peer.go +++ b/internal/api/stremio_peer.go @@ -3,6 +3,7 @@ package api import ( "context" "encoding/json" + "errors" "log/slog" "net/url" "path/filepath" @@ -41,6 +42,19 @@ type PeerPlayback interface { Play(ctx context.Context, req PeerPlayRequest) (PeerPlayResult, error) } +// PeerRetryError is returned by PeerPlayback.Play when the release cannot be +// fetched right now for a temporary reason (a rate limit, a publisher back-off). +// After is the suggested wait in seconds. +type PeerRetryError struct { + After int + Err error +} + +func (e *PeerRetryError) Error() string { + return "peer release temporarily unavailable: " + e.Err.Error() +} +func (e *PeerRetryError) Unwrap() error { return e.Err } + type peerPlaybackHolder struct{ p PeerPlayback } // SetPeerPlayback installs (or, with nil, removes) the peer network's Stremio @@ -294,6 +308,13 @@ func (s *Server) handleStremioPeerPlay(c *fiber.Ctx) error { return h.p.Play(workCtx, req) }) if err != nil { + var retry *PeerRetryError + if errors.As(err, &retry) { + // A fetch limit or a publisher back-off is temporary: say so. + slog.InfoContext(ctx, "Peer network play deferred", "retry_after", retry.After) + c.Set("Retry-After", strconv.Itoa(max(retry.After, 1))) + return RespondError(c, fiber.StatusTooManyRequests, "RATE_LIMITED", "Peer release is not available right now", "try again shortly") + } slog.WarnContext(ctx, "Peer network play failed", "error", err) return RespondServiceUnavailable(c, "Failed to prepare peer release", "the release could not be verified here") } diff --git a/internal/api/stremio_peer_test.go b/internal/api/stremio_peer_test.go index 9789f3afb..a9a82fcf6 100644 --- a/internal/api/stremio_peer_test.go +++ b/internal/api/stremio_peer_test.go @@ -434,3 +434,14 @@ func TestStreamList_MergeWithoutPeerStreamsIsByteIdenticalToTheFormerOutput(t *t require.Len(t, merged, 3) assert.Equal(t, "P", merged[1]["name"]) } + +// A fetch rate limit or publisher back-off is not "the release could not be +// verified": the player is told to retry later (429 + Retry-After). +func TestPeerPlay_RateLimitedFetchAnswers429WithRetryAfter(t *testing.T) { + f := &fakePlayback{offers: map[string]bool{relA: true}, playErr: &PeerRetryError{After: 120, Err: assertErr("secret detail 10.0.0.5")}} + h := newPeerHarness(t, f) + resp := h.get(t, h.key, relA, "type=movie&id=tt1") + assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode) + assert.Equal(t, "120", resp.Header.Get("Retry-After")) + assert.NotContains(t, readAll(t, resp), "10.0.0.5") +} diff --git a/internal/database/migrations/postgres/063_add_peernet_connected_ip_and_failed_bundles.sql b/internal/database/migrations/postgres/063_add_peernet_connected_ip_and_failed_bundles.sql new file mode 100644 index 000000000..e231bc12c --- /dev/null +++ b/internal/database/migrations/postgres/063_add_peernet_connected_ip_and_failed_bundles.sql @@ -0,0 +1,28 @@ +-- +goose Up +-- connected_ip is the remote IP this node actually connected to when it last +-- verified the peer (a completed mutual-TLS contact). Corroboration counts +-- distinct /24 (IPv4) or /48 (IPv6) ranges of these addresses, never of the +-- stored address string, which may be a host name: two names for one server +-- are one range. A peer with no recorded connected IP does not count. +ALTER TABLE peernet_peers ADD COLUMN connected_ip TEXT NOT NULL DEFAULT ''; + +-- bundle_id is the Bundle ID of the bundle an import was accepted from, so a +-- later failed full check can blacklist exactly that layout. +ALTER TABLE peernet_bundle_imports ADD COLUMN bundle_id TEXT NOT NULL DEFAULT ''; + +-- Bundle IDs whose layout failed local verification, kept for a while so the +-- same bad layout offered by another publisher is not accepted again. Rows +-- are pruned after their expiry (7 days). Persisted: a failure surfaces up to +-- a week after the import, across restarts. failed_at is Unix seconds. +CREATE TABLE peernet_failed_bundles ( + bundle_id TEXT NOT NULL PRIMARY KEY, + release_id TEXT NOT NULL, + failed_at BIGINT NOT NULL +); +CREATE INDEX idx_peernet_failed_bundles_failed_at ON peernet_failed_bundles(failed_at); + +-- +goose Down +DROP INDEX IF EXISTS idx_peernet_failed_bundles_failed_at; +DROP TABLE IF EXISTS peernet_failed_bundles; +ALTER TABLE peernet_bundle_imports DROP COLUMN bundle_id; +ALTER TABLE peernet_peers DROP COLUMN connected_ip; diff --git a/internal/database/migrations/sqlite/063_add_peernet_connected_ip_and_failed_bundles.sql b/internal/database/migrations/sqlite/063_add_peernet_connected_ip_and_failed_bundles.sql new file mode 100644 index 000000000..39cffd843 --- /dev/null +++ b/internal/database/migrations/sqlite/063_add_peernet_connected_ip_and_failed_bundles.sql @@ -0,0 +1,28 @@ +-- +goose Up +-- connected_ip is the remote IP this node actually connected to when it last +-- verified the peer (a completed mutual-TLS contact). Corroboration counts +-- distinct /24 (IPv4) or /48 (IPv6) ranges of these addresses, never of the +-- stored address string, which may be a host name: two names for one server +-- are one range. A peer with no recorded connected IP does not count. +ALTER TABLE peernet_peers ADD COLUMN connected_ip TEXT NOT NULL DEFAULT ''; + +-- bundle_id is the Bundle ID of the bundle an import was accepted from, so a +-- later failed full check can blacklist exactly that layout. +ALTER TABLE peernet_bundle_imports ADD COLUMN bundle_id TEXT NOT NULL DEFAULT ''; + +-- Bundle IDs whose layout failed local verification, kept for a while so the +-- same bad layout offered by another publisher is not accepted again. Rows +-- are pruned after their expiry (7 days). Persisted: a failure surfaces up to +-- a week after the import, across restarts. failed_at is Unix seconds. +CREATE TABLE peernet_failed_bundles ( + bundle_id TEXT NOT NULL PRIMARY KEY, + release_id TEXT NOT NULL, + failed_at INTEGER NOT NULL +); +CREATE INDEX idx_peernet_failed_bundles_failed_at ON peernet_failed_bundles(failed_at); + +-- +goose Down +DROP INDEX IF EXISTS idx_peernet_failed_bundles_failed_at; +DROP TABLE IF EXISTS peernet_failed_bundles; +ALTER TABLE peernet_bundle_imports DROP COLUMN bundle_id; +ALTER TABLE peernet_peers DROP COLUMN connected_ip; diff --git a/internal/database/peernet_bundle_import_repository.go b/internal/database/peernet_bundle_import_repository.go index e93b625e6..fd72c5e81 100644 --- a/internal/database/peernet_bundle_import_repository.go +++ b/internal/database/peernet_bundle_import_repository.go @@ -343,6 +343,26 @@ func (r *PeernetBundleImportRepository) FailHeldQueueItem(ctx context.Context, q return true, nil } +// RequeueHeldQueueItem puts a held queue item back as pending, with its +// provisional history gone, so it is imported again from its NZB. It reports +// whether an item was held and changed. +func (r *PeernetBundleImportRepository) RequeueHeldQueueItem(ctx context.Context, queueID int64) (bool, error) { + now := time.Now().UTC() + res, err := r.db.ExecContext(ctx, `UPDATE import_queue SET status = 'pending', error_message = NULL, peer_hold = 0, + started_at = NULL, completed_at = NULL, updated_at = ? WHERE id = ? AND peer_hold <> 0`, now, queueID) + if err != nil { + return false, fmt.Errorf("peernet bundle import: requeue held item: %w", err) + } + n, err := res.RowsAffected() + if err != nil || n == 0 { + return false, err + } + if _, err := r.db.ExecContext(ctx, `DELETE FROM import_history WHERE nzb_id = ?`, queueID); err != nil { + return true, fmt.Errorf("peernet bundle import: drop history of a requeued item: %w", err) + } + return true, nil +} + // SetQueue links the rows of the given paths to a queue item. func (r *PeernetBundleImportRepository) SetQueue(ctx context.Context, paths []string, queueID int64) error { for len(paths) > 0 { @@ -513,3 +533,97 @@ func (r *PeernetBundleImportRepository) OrphanedHolds(ctx context.Context) ([]in } return out, rows.Err() } + +// SettledHolds lists the held queue items whose import is finished and owes +// nobody anything: every row is passed and committed and no ARR is to be told. +// Such a hold can only still be set because releasing it failed once. +func (r *PeernetBundleImportRepository) SettledHolds(ctx context.Context) ([]int64, error) { + rows, err := r.db.QueryContext(ctx, `SELECT q.id FROM import_queue q + WHERE q.peer_hold <> 0 + AND EXISTS (SELECT 1 FROM peernet_bundle_imports b WHERE b.queue_id = q.id) + AND NOT EXISTS (SELECT 1 FROM peernet_bundle_imports b WHERE b.queue_id = q.id + AND (b.state <> 'passed' OR b.committed = 0 OR b.arr_owed <> 0)) + ORDER BY q.id LIMIT 1000`) + if err != nil { + return nil, fmt.Errorf("peernet bundle import: settled holds: %w", err) + } + defer func() { _ = rows.Close() }() + var out []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + out = append(out, id) + } + return out, rows.Err() +} + +// SetBundleID records the Bundle ID the rows of the given paths were accepted +// from. +func (r *PeernetBundleImportRepository) SetBundleID(ctx context.Context, paths []string, bundleID string) error { + for len(paths) > 0 { + n := min(len(paths), maxBundleImportBatch) + args := []any{bundleID} + for _, p := range paths[:n] { + args = append(args, normalizeHealthPath(p)) + } + paths = paths[n:] + if _, err := r.db.ExecContext(ctx, `UPDATE peernet_bundle_imports SET bundle_id = ? WHERE file_path IN (`+inList(n)+`)`, args...); err != nil { + return fmt.Errorf("peernet bundle import: set bundle id: %w", err) + } + } + return nil +} + +// BundleIDOf returns the Bundle ID a row was accepted from ("" if unknown). +func (r *PeernetBundleImportRepository) BundleIDOf(ctx context.Context, filePath string) (string, error) { + var id string + err := r.db.QueryRowContext(ctx, `SELECT bundle_id FROM peernet_bundle_imports WHERE file_path = ?`, normalizeHealthPath(filePath)).Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("peernet bundle import: bundle id: %w", err) + } + return id, nil +} + +// maxFailedBundles bounds the negative cache: the oldest entries go first. +const maxFailedBundles = 10_000 + +// RecordFailedBundle puts a Bundle ID whose layout failed local verification on +// the negative cache (the same layout from another publisher is then skipped). +func (r *PeernetBundleImportRepository) RecordFailedBundle(ctx context.Context, bundleID, releaseID string, at time.Time) error { + if bundleID == "" { + return nil + } + if _, err := r.db.ExecContext(ctx, `INSERT INTO peernet_failed_bundles (bundle_id, release_id, failed_at) VALUES (?, ?, ?) + ON CONFLICT (bundle_id) DO UPDATE SET failed_at = excluded.failed_at`, bundleID, releaseID, at.Unix()); err != nil { + return fmt.Errorf("peernet bundle import: record failed bundle: %w", err) + } + // Bounded: keep the newest maxFailedBundles. + if _, err := r.db.ExecContext(ctx, `DELETE FROM peernet_failed_bundles WHERE bundle_id IN ( + SELECT bundle_id FROM peernet_failed_bundles ORDER BY failed_at DESC LIMIT 1000000 OFFSET ?)`, maxFailedBundles); err != nil { + return fmt.Errorf("peernet bundle import: trim failed bundles: %w", err) + } + return nil +} + +// IsFailedBundle reports whether the Bundle ID failed local verification at or +// after notBefore. +func (r *PeernetBundleImportRepository) IsFailedBundle(ctx context.Context, bundleID string, notBefore time.Time) (bool, error) { + var n int + err := r.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM peernet_failed_bundles WHERE bundle_id = ? AND failed_at >= ?`, + bundleID, notBefore.Unix()).Scan(&n) + if err != nil { + return false, fmt.Errorf("peernet bundle import: failed bundle lookup: %w", err) + } + return n > 0, nil +} + +// PruneFailedBundles removes entries older than before. +func (r *PeernetBundleImportRepository) PruneFailedBundles(ctx context.Context, before time.Time) error { + _, err := r.db.ExecContext(ctx, `DELETE FROM peernet_failed_bundles WHERE failed_at < ?`, before.Unix()) + return err +} diff --git a/internal/database/peernet_bundle_import_repository_test.go b/internal/database/peernet_bundle_import_repository_test.go index 682c9ca61..2916160a3 100644 --- a/internal/database/peernet_bundle_import_repository_test.go +++ b/internal/database/peernet_bundle_import_repository_test.go @@ -247,3 +247,29 @@ func TestPeernetBundleImport_MoveDeleteAndPendingCounts(t *testing.T) { assert.True(t, ok) }) } + +func TestPeerBundleImport_BundleIDAndFailedBundleCache(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + require.NoError(t, r.Upsert(ctx, []PeernetBundleImport{bundleRow("m/a.mkv", 5)})) + require.NoError(t, r.SetBundleID(ctx, []string{"m/a.mkv"}, hex64("e"))) + got, err := r.BundleIDOf(ctx, "m/a.mkv") + require.NoError(t, err) + assert.Equal(t, hex64("e"), got) + + now := time.Now() + bad, err := r.IsFailedBundle(ctx, hex64("e"), now.Add(-time.Hour)) + require.NoError(t, err) + assert.False(t, bad) + require.NoError(t, r.RecordFailedBundle(ctx, hex64("e"), hex64("b"), now)) + bad, err = r.IsFailedBundle(ctx, hex64("e"), now.Add(-time.Hour)) + require.NoError(t, err) + assert.True(t, bad) + bad, err = r.IsFailedBundle(ctx, hex64("e"), now.Add(time.Hour)) + require.NoError(t, err) + assert.False(t, bad, "an entry older than the window no longer counts") + require.NoError(t, r.PruneFailedBundles(ctx, now.Add(time.Hour))) + bad, _ = r.IsFailedBundle(ctx, hex64("e"), now.Add(-time.Hour)) + assert.False(t, bad, "pruned") + }) +} diff --git a/internal/database/peernet_bundle_queries.go b/internal/database/peernet_bundle_queries.go index 5b575e43e..f9c081122 100644 --- a/internal/database/peernet_bundle_queries.go +++ b/internal/database/peernet_bundle_queries.go @@ -169,3 +169,27 @@ func (r *PeernetPublishStateRepository) CountAnnouncedBundles(ctx context.Contex } return n, nil } + +// ListUnpublishedReleases returns up to limit release IDs of the content index +// that have no release-record publish state on network: releases that became +// eligible while the network did not publish (level below 2). +func (r *PeernetPublishStateRepository) ListUnpublishedReleases(ctx context.Context, network string, limit int) ([]string, error) { + rows, err := r.db.QueryContext(ctx, `SELECT DISTINCT c.release_id FROM peernet_content_index c + WHERE c.release_id <> '' + AND NOT EXISTS (SELECT 1 FROM peernet_publish_state s + WHERE s.network_id = ? AND s.kind = ? AND s.release_id = c.release_id) + ORDER BY c.release_id LIMIT ?`, network, PeernetRecordKindRelease, limit) + if err != nil { + return nil, fmt.Errorf("peernet publish state: list unpublished releases: %w", err) + } + defer func() { _ = rows.Close() }() + var out []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + out = append(out, id) + } + return out, rows.Err() +} diff --git a/internal/database/peernet_peer_repository.go b/internal/database/peernet_peer_repository.go index 232557d88..cc862d3f2 100644 --- a/internal/database/peernet_peer_repository.go +++ b/internal/database/peernet_peer_repository.go @@ -274,6 +274,25 @@ func peerAddrPrefix(address string) string { return pre.String() } +// connectedPrefix is the /24 (IPv4) or /48 (IPv6) of a literal IP, "" for +// anything else (an empty or non-IP value counts for nothing). +func connectedPrefix(ip string) string { + a, err := netip.ParseAddr(strings.TrimSpace(ip)) + if err != nil { + return "" + } + a = a.Unmap() + bits := 48 + if a.Is4() { + bits = 24 + } + pre, err := a.Prefix(bits) + if err != nil { + return "" + } + return pre.String() +} + // trimUnverifiedPrefix makes room for one more unverified learned row in p's // address prefix by evicting that prefix's stalest unverified rows, so one // address range can only ever churn its own rows. @@ -611,19 +630,41 @@ func (r *PeernetPeerRepository) KnownAmong(ctx context.Context, network string, } // KnownPrefixes is KnownAmong that also returns, per known node, the network -// prefix (/24 IPv4, /48 IPv6) of the address this node verified it at. Nodes -// whose address yields no prefix are left out: they cannot be told apart from -// each other and so count as nothing. +// prefix (/24 IPv4, /48 IPv6) of the remote IP this node actually connected to +// when it last verified it. The stored address string is never used: a host +// name would let one server count as several ranges. Nodes with no recorded +// connected IP are left out: they count as nothing. func (r *PeernetPeerRepository) KnownPrefixes(ctx context.Context, network string, nodes []string, d time.Duration, now time.Time) (map[string]string, error) { out := map[string]string{} - err := r.knownRows(ctx, network, nodes, d, now, func(node, address string) { - if p := peerAddrPrefix(address); address != "" && p != "" { + err := r.knownRows(ctx, network, nodes, d, now, func(node, connected string) { + if p := connectedPrefix(connected); p != "" { out[node] = p } }) return out, err } +// RecordConnectedIP stores the remote IP this node actually connected to when +// it last verified the peer. Corroboration keys on it (see KnownPrefixes). +func (r *PeernetPeerRepository) RecordConnectedIP(ctx context.Context, network, node, ip string) error { + _, err := r.exec(ctx, "record peernet peer connected ip", + `UPDATE peernet_peers SET connected_ip = ? WHERE network_id = ? AND node_id = ?`, ip, network, node) + return err +} + +// ConnectedIP returns the recorded connected IP of a peer ("" if none). +func (r *PeernetPeerRepository) ConnectedIP(ctx context.Context, network, node string) (string, error) { + var ip string + err := r.db.QueryRowContext(ctx, `SELECT connected_ip FROM peernet_peers WHERE network_id = ? AND node_id = ?`, network, node).Scan(&ip) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("peernet peer connected ip: %w", err) + } + return ip, nil +} + func (r *PeernetPeerRepository) knownRows(ctx context.Context, network string, nodes []string, d time.Duration, now time.Time, each func(node, address string)) error { const chunk = 200 for start := 0; start < len(nodes); start += chunk { @@ -633,7 +674,7 @@ func (r *PeernetPeerRepository) knownRows(ctx context.Context, network string, n for _, n := range part { args = append(args, n) } - q := `SELECT node_id, address FROM peernet_peers WHERE network_id = ? AND verified = 1 + q := `SELECT node_id, connected_ip FROM peernet_peers WHERE network_id = ? AND verified = 1 AND first_verified > 0 AND first_verified <= ? AND last_success >= ? AND node_id IN (?` + strings.Repeat(",?", len(part)-1) + `)` rows, err := r.db.QueryContext(ctx, q, args...) diff --git a/internal/database/peernet_peer_repository_test.go b/internal/database/peernet_peer_repository_test.go index c6c5d799c..3fbf1275a 100644 --- a/internal/database/peernet_peer_repository_test.go +++ b/internal/database/peernet_peer_repository_test.go @@ -4,7 +4,9 @@ import ( "context" "errors" "fmt" + "net" "path/filepath" + "strings" "testing" "time" @@ -143,11 +145,40 @@ func TestPeerRepo_KnownPrefixesReportTheVerifiedAddressPrefix(t *testing.T) { require.NoError(t, r.RecordSuccess(ctx, p.NetworkID, p.NodeID, "", peerT0, peerT0.Add(time.Hour))) } require.NoError(t, r.RecordSuccess(ctx, young.NetworkID, young.NodeID, "", peerT0.Add(20*time.Hour), peerT0.Add(21*time.Hour))) + for _, p := range []PeernetPeer{a, b, v6} { + host, _, _ := net.SplitHostPort(p.Address) + require.NoError(t, r.RecordConnectedIP(ctx, p.NetworkID, p.NodeID, strings.Trim(host, "[]"))) + } got, err := r.KnownPrefixes(ctx, a.NetworkID, []string{a.NodeID, b.NodeID, v6.NodeID, young.NodeID, nodeHex(9)}, 24*time.Hour, peerT0.Add(25*time.Hour)) require.NoError(t, err) assert.Equal(t, map[string]string{a.NodeID: "8.8.8.0/24", b.NodeID: "8.8.8.0/24", v6.NodeID: "2001:db8:1::/48"}, got) } +// Two DNS names for one server are one range: corroboration keys on the IP the +// node actually connected to, never on the stored address string. +func TestPeerRepo_KnownPrefixesKeyOnTheConnectedIPNotTheName(t *testing.T) { + r := newPeerTestRepo(t) + ctx := context.Background() + a, b, c, d := mkPeer(1, PeernetSourceExchange, peerT0), mkPeer(2, PeernetSourceExchange, peerT0), + mkPeer(3, PeernetSourceExchange, peerT0), mkPeer(4, PeernetSourceExchange, peerT0) + a.Address, b.Address, c.Address, d.Address = "one.example.org:7000", "two.example.org:7000", "[2001:db8::9]:7000", "ip.example.org:7000" + for _, p := range []PeernetPeer{a, b, c, d} { + _, _ = r.Insert(ctx, p, PeernetPeerCaps{}) + require.NoError(t, r.RecordSuccess(ctx, p.NetworkID, p.NodeID, "", peerT0, peerT0.Add(time.Hour))) + } + require.NoError(t, r.RecordConnectedIP(ctx, a.NetworkID, a.NodeID, "203.0.113.7")) + require.NoError(t, r.RecordConnectedIP(ctx, b.NetworkID, b.NodeID, "203.0.113.200")) + require.NoError(t, r.RecordConnectedIP(ctx, c.NetworkID, c.NodeID, "2001:db8:1:2::5")) + // d never recorded a connected IP. + got, err := r.KnownPrefixes(ctx, a.NetworkID, []string{a.NodeID, b.NodeID, c.NodeID, d.NodeID}, 24*time.Hour, peerT0.Add(25*time.Hour)) + require.NoError(t, err) + assert.Equal(t, map[string]string{a.NodeID: "203.0.113.0/24", b.NodeID: "203.0.113.0/24", c.NodeID: "2001:db8:1::/48"}, got, + "two names, one /24; a peer without a connected IP counts for nothing") + ip, err := r.ConnectedIP(ctx, a.NetworkID, a.NodeID) + require.NoError(t, err) + assert.Equal(t, "203.0.113.7", ip) +} + func TestPeerRepo_PerSourceCapEvictsStalestUnverifiedOfSource(t *testing.T) { r := newPeerTestRepo(t) ctx := context.Background() diff --git a/internal/database/peernet_queue_hold_test.go b/internal/database/peernet_queue_hold_test.go index f33e7e403..0c1da0510 100644 --- a/internal/database/peernet_queue_hold_test.go +++ b/internal/database/peernet_queue_hold_test.go @@ -120,6 +120,46 @@ func TestPeerHold_FailedIsFailedInHistoryWithMessage(t *testing.T) { }) } +func TestPeerHold_RequeueTurnsAHeldItemIntoAPendingOneWithoutHistory(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + f := newHoldFixture(t, r, "requeue", false) + changed, err := r.RequeueHeldQueueItem(ctx, f.id) + require.NoError(t, err) + assert.False(t, changed, "only a held item can be requeued") + + require.NoError(t, r.SetQueueHold(ctx, f.id, true)) + changed, err = r.RequeueHeldQueueItem(ctx, f.id) + require.NoError(t, err) + require.True(t, changed) + got, err := f.repo.GetQueueItem(ctx, f.id) + require.NoError(t, err) + assert.Equal(t, QueueStatusPending, got.Status) + assert.False(t, got.PeerHold) + assert.NotContains(t, f.historyStatuses(t), f.id, "the provisional history is gone") + }) +} + +func TestPeerHold_SettledHoldsAreOnlyFinishedImportsOwingNoARR(t *testing.T) { + forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { + ctx := context.Background() + f := newHoldFixture(t, r, "settled", false) + require.NoError(t, r.SetQueueHold(ctx, f.id, true)) + row := bundleRow("m/a.mkv", f.id) + row.ArrOwed = false + require.NoError(t, r.Upsert(ctx, []PeernetBundleImport{row})) + ids, err := r.SettledHolds(ctx) + require.NoError(t, err) + assert.Empty(t, ids, "a pending, uncommitted row is not settled") + _, err = r.MarkPassed(ctx, "m/a.mkv") + require.NoError(t, err) + require.NoError(t, r.CommitRows(ctx, []string{"m/a.mkv"})) + ids, err = r.SettledHolds(ctx) + require.NoError(t, err) + assert.Equal(t, []int64{f.id}, ids) + }) +} + func TestPeerHold_UserSkipStaysHidden(t *testing.T) { forEachBundleImportBackend(t, func(t *testing.T, r *PeernetBundleImportRepository) { ctx := context.Background() diff --git a/internal/database/peernet_sync_cursor_repository.go b/internal/database/peernet_sync_cursor_repository.go index 3bff21c95..c9d8f3e97 100644 --- a/internal/database/peernet_sync_cursor_repository.go +++ b/internal/database/peernet_sync_cursor_repository.go @@ -174,3 +174,22 @@ func (r *PeernetSyncCursorRepository) DeleteOrphans(ctx context.Context) (int64, } return rowsAffected(res, "delete orphan peernet cursors") } + +// LastPullAt returns when a cursor of the network that had pulled records was +// last updated (zero: none). It lets the status show the last pull after a +// restart; a push updates the same row, so it is the last sync activity of a +// peer we pulled from. +func (r *PeernetSyncCursorRepository) LastPullAt(ctx context.Context, networkID string) (time.Time, error) { + if !isLowerHexID(networkID) { + return time.Time{}, errors.New("peernet cursors: invalid network id") + } + var at sql.NullInt64 + err := r.db.QueryRowContext(ctx, `SELECT MAX(updated_at) FROM peernet_sync_cursors WHERE network_id = ? AND pulled_seq > 0`, networkID).Scan(&at) + if err != nil { + return time.Time{}, fmt.Errorf("peernet cursors: last pull: %w", err) + } + if !at.Valid { + return time.Time{}, nil + } + return timeOrZero(at.Int64), nil +} diff --git a/internal/database/repository.go b/internal/database/repository.go index 3352e7556..933f3125e 100644 --- a/internal/database/repository.go +++ b/internal/database/repository.go @@ -420,10 +420,27 @@ func (r *Repository) UpdateQueueItemIndexer(ctx context.Context, id int64, index return nil } +// dropHeldHistory deletes the provisional history rows of the held peer-import +// queue items matching the condition. A held item is not yet a finished import, +// so nothing of it may outlive its queue row. +func (r *Repository) dropHeldHistory(ctx context.Context, cond string, args ...any) error { + _, err := r.db.ExecContext(ctx, + `DELETE FROM import_history WHERE nzb_id IN (SELECT id FROM import_queue WHERE peer_hold <> 0 AND `+cond+`)`, args...) + if err != nil { + return fmt.Errorf("failed to drop the provisional history of a held peer import: %w", err) + } + return nil +} + // RemoveFromQueueByDownloadID removes an item from the queue by its DownloadID func (r *Repository) RemoveFromQueueByDownloadID(ctx context.Context, downloadID string) error { query := `DELETE FROM import_queue WHERE download_id = ?` + // The provisional history of a held peer import goes first: once the queue + // row is gone its history row loses nzb_id and would show as Completed. + if err := r.dropHeldHistory(ctx, `download_id = ?`, downloadID); err != nil { + return err + } result, err := r.db.ExecContext(ctx, query, downloadID) if err != nil { return fmt.Errorf("failed to remove from queue by download_id: %w", err) @@ -497,6 +514,9 @@ func (r *Repository) HasActiveOrRecentQueueItemForStoragePath( // DeleteQueueItemsByPath removes items from the queue matching the given path func (r *Repository) DeleteQueueItemsByPath(ctx context.Context, path string) error { + if err := r.dropHeldHistory(ctx, `(storage_path = ? OR nzb_path = ?)`, path, path); err != nil { + return err + } _, err := r.db.ExecContext(ctx, `DELETE FROM import_queue WHERE storage_path = ? OR nzb_path = ?`, path, path) if err != nil { diff --git a/internal/health/checker.go b/internal/health/checker.go index e99eba48c..fe366dff8 100644 --- a/internal/health/checker.go +++ b/internal/health/checker.go @@ -123,6 +123,8 @@ type HealthChecker struct { // peerAdviser supplies peer-network advice for scheduled re-checks (see // peer_advice.go). Nil by default: no behaviour change and no overhead. peerAdviser atomic.Pointer[peerAdviserHolder] + // peerHeaders is the optional header sample of a peer import's full check. + peerHeaders atomic.Pointer[peerHeaderHolder] // patchStats is set while a healthy observer is installed: only then is // CheckStats.PatchedByPAR2 computed (see patch_summary.go). patchStats atomic.Bool @@ -184,7 +186,9 @@ type healthCheckInput struct { // ID strings survive past preparation, so the proto segment slice is // collectible before the network sweep begins. type preparedCheck struct { - filePath string + filePath string + // fullCheck marks a file that owes a forced full check (a peer import). + fullCheck bool sourceNzbPath string sampledIDs []string earlyEvent *HealthEvent @@ -307,6 +311,7 @@ func (hc *HealthChecker) prepareCheck(ctx context.Context, filePath string, opts // Override sample percentage if forced full check is requested if len(opts) > 0 && (opts[0].ForceFullCheck || owesFullCheck(opts[0], filePath)) { + prep.fullCheck = len(opts) > 0 && owesFullCheck(opts[0], filePath) samplePercentage = 100 slog.InfoContext(ctx, "Forcing full health check (100% sampling)", "file_path", filePath) } @@ -436,6 +441,13 @@ func (hc *HealthChecker) judgeValidation(ctx context.Context, prep preparedCheck event.Details = healthyDetails } + // A peer import also gets a header sample (optional hook): the STAT sweep + // cannot see a size or offset lie. + if bad := hc.verifyPeerHeaders(ctx, prep, event); bad != nil { + bad.Stats = event.Stats + return *bad + } + // Only a consumer of the notice needs the patch summary, and only for a // healthy result, so the store is scanned here and nowhere else. if hc.patchStats.Load() { diff --git a/internal/health/peer_bundle.go b/internal/health/peer_bundle.go index fdbfe5616..3fb7cea22 100644 --- a/internal/health/peer_bundle.go +++ b/internal/health/peer_bundle.go @@ -5,6 +5,7 @@ import ( "log/slog" "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/holes" ) // PeerBundleGate is what the health worker needs to know about imports that @@ -34,15 +35,34 @@ type PeerBundleGate interface { FullCheckFailed(ctx context.Context, filePath string) } +// PeerBundleDegradedGate is optionally implemented by a gate that wants to hear +// about a degraded verdict (the file would play with a few missing segments) +// separately from other failures. A pending peer import never ends degraded: it +// fails at once, and an implementation that does not implement this interface +// simply sees FullCheckFailed. +type PeerBundleDegradedGate interface { + FullCheckDegraded(ctx context.Context, filePath string) +} + type peerBundleGateHolder struct{ g PeerBundleGate } // SetPeerBundleGate installs (or, with nil, removes) the gate. Safe at any time. +// +// A gate that also implements PeerHeaderVerifier gets the header sample of the +// forced full check; removing the gate removes the verifier. func (hw *HealthWorker) SetPeerBundleGate(g PeerBundleGate) { if g == nil { hw.peerBundleGate.Store(nil) + if hw.healthChecker != nil { + hw.healthChecker.SetPeerHeaderVerifier(nil) + } return } hw.peerBundleGate.Store(&peerBundleGateHolder{g: g}) + if hw.healthChecker != nil { + v, _ := g.(PeerHeaderVerifier) + hw.healthChecker.SetPeerHeaderVerifier(v) + } } // pendingPeerBundles returns which paths owe a full check; nil with no gate. @@ -83,8 +103,15 @@ func (hw *HealthWorker) notifyPeerBundleResults(ctx context.Context, pending map switch { case u.Type == database.UpdateTypeHealthy && ev.Type == EventTypeFileHealthy: h.g.FullCheckPassed(ctx, u.FilePath) - case ev.Type == EventTypeFileCorrupted && u.Type != database.UpdateTypeRetry && + case (ev.Type == EventTypeFileCorrupted || ev.Type == EventTypeCheckFailed) && u.Type != database.UpdateTypeRetry && u.Type != database.UpdateTypeInconclusive && u.Type != database.UpdateTypeDeferred && !u.Skip: + // A check that failed outright and ended as a corrupted status is + // a verdict like any other; a degraded one (playable with a few + // missing segments) is told apart so the fast path can fall back. + if g, ok := h.g.(PeerBundleDegradedGate); ok && ev.Classification != nil && ev.Classification.Verdict == holes.VerdictDegraded { + g.FullCheckDegraded(ctx, u.FilePath) + continue + } h.g.FullCheckFailed(ctx, u.FilePath) } } diff --git a/internal/health/peer_headers.go b/internal/health/peer_headers.go new file mode 100644 index 000000000..823dafa06 --- /dev/null +++ b/internal/health/peer_headers.go @@ -0,0 +1,88 @@ +package health + +import ( + "context" + "errors" + "fmt" + "log/slog" + + "github.com/kipsilabs/altmount/internal/database" +) + +// ErrPeerLayoutMismatch is what a PeerHeaderVerifier returns when the yEnc +// headers of sampled articles contradict the file's recorded layout. +var ErrPeerLayoutMismatch = errors.New("article headers contradict the imported layout") + +// layoutMismatch is implemented by an error a verifier returns for a +// contradiction without importing this package's sentinel. +type layoutMismatch interface{ PeerLayoutMismatch() bool } + +func isLayoutMismatch(err error) bool { + var m layoutMismatch + return errors.Is(err, ErrPeerLayoutMismatch) || (errors.As(err, &m) && m.PeerLayoutMismatch()) +} + +// PeerHeaderVerifier is an optional extra check of a pending peer import's +// forced full check. The STAT sweep proves the articles exist; this reads the +// yEnc headers of a further random sample (part size and offset against the +// layout), which is sampling, not proof. It returns nil when the sample +// agrees, an error wrapping ErrPeerLayoutMismatch (or implementing PeerLayoutMismatch() bool) when it contradicts the +// layout, and any other error when it could not be completed. The interface +// is defined here so this package never imports the peer network; with no +// verifier installed nothing changes. +type PeerHeaderVerifier interface { + VerifyPeerHeaders(ctx context.Context, filePath string) error +} + +type peerHeaderHolder struct{ v PeerHeaderVerifier } + +// SetPeerHeaderVerifier installs (or, with nil, removes) the verifier. +func (hc *HealthChecker) SetPeerHeaderVerifier(v PeerHeaderVerifier) { + if v == nil { + hc.peerHeaders.Store(nil) + return + } + hc.peerHeaders.Store(&peerHeaderHolder{v: v}) +} + +// SetPeerHeaderVerifier installs the verifier on the worker's checker. +func (hw *HealthWorker) SetPeerHeaderVerifier(v PeerHeaderVerifier) { + hw.healthChecker.SetPeerHeaderVerifier(v) +} + +// verifyPeerHeaders runs the verifier for a file owing a full check. A nil +// event means the sample agreed (or there is no verifier). +func (hc *HealthChecker) verifyPeerHeaders(ctx context.Context, prep preparedCheck, base HealthEvent) *HealthEvent { + h := hc.peerHeaders.Load() + if h == nil || !prep.fullCheck { + return nil + } + var err error + func() { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("header verifier panicked: %v", r) + } + }() + err = h.v.VerifyPeerHeaders(ctx, prep.filePath) + }() + switch { + case err == nil: + return nil + case isLayoutMismatch(err): + slog.WarnContext(ctx, "Peer import: article headers contradict the layout", "file_path", prep.filePath, "error", err) + base.Type = EventTypeFileCorrupted + base.Status = database.HealthStatusCorrupted + base.Error = err + details := database.HealthErrorDetails{ErrorType: "peer_layout_mismatch", Message: err.Error()} + base.Details = details.Marshal() + return &base + default: + base.Type = EventTypeCheckInconclusive + base.Status = database.HealthStatusPending + base.Error = fmt.Errorf("peer header check inconclusive: %w", err) + details := database.HealthErrorDetails{ErrorType: "check_inconclusive", Message: base.Error.Error()} + base.Details = details.Marshal() + return &base + } +} diff --git a/internal/health/peer_verdicts_test.go b/internal/health/peer_verdicts_test.go new file mode 100644 index 000000000..a47124317 --- /dev/null +++ b/internal/health/peer_verdicts_test.go @@ -0,0 +1,110 @@ +package health + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/holes" +) + +// degradedGate is a gate that also wants to hear about degraded verdicts. +type degradedGate struct { + fakeGate + mu2 sync.Mutex + degraded []string +} + +func (g *degradedGate) FullCheckDegraded(_ context.Context, p string) { + g.mu2.Lock() + g.degraded = append(g.degraded, p) + g.mu2.Unlock() +} + +// A check that failed with no segment data and ended as a corrupted status +// is a definitive failure like any corrupted verdict. +func TestPeerBundle_CheckFailedEndingCorruptedFailsTheImport(t *testing.T) { + const p = "m/a.mkv" + hw := &HealthWorker{} + gate := &fakeGate{} + hw.SetPeerBundleGate(gate) + hw.notifyPeerBundleResults(context.Background(), map[string]struct{}{p: {}}, + []database.HealthStatusUpdate{{FilePath: p, Type: database.UpdateTypeCorrupted}}, []bool{true}, + map[string]HealthEvent{p: {Type: EventTypeCheckFailed, FilePath: p}}) + assert.Equal(t, []string{p}, gate.failed) +} + +// A degraded verdict (playable with a few missing segments) ends a pending +// peer import: the gate is told it was degraded, never that it passed. +func TestPeerBundle_DegradedVerdictEndsThePendingImport(t *testing.T) { + const p = "m/a.mkv" + hw := &HealthWorker{} + gate := °radedGate{} + hw.SetPeerBundleGate(gate) + ev := HealthEvent{Type: EventTypeFileCorrupted, FilePath: p, Classification: &holes.Impact{Verdict: holes.VerdictDegraded}} + hw.notifyPeerBundleResults(context.Background(), map[string]struct{}{p: {}}, + []database.HealthStatusUpdate{{FilePath: p, Type: database.UpdateTypeCorrupted}}, []bool{true}, + map[string]HealthEvent{p: ev}) + assert.Equal(t, []string{p}, gate.degraded) + assert.Empty(t, gate.failed) + assert.Empty(t, gate.passed) +} + +func TestPeerBundle_DegradedVerdictOfAPendingImportIsNotRecordedAsDegraded(t *testing.T) { + env := newRepairTestEnv(t, t.TempDir(), nil) + const filePath = "/movies/movie.mp4" + env.hw.SetPeerBundleGate(&fakeGate{pending: map[string]struct{}{filePath: {}}}) + fh := database.FileHealth{FilePath: filePath, Status: database.HealthStatusPending} + ev := HealthEvent{Type: EventTypeFileCorrupted, FilePath: filePath, Status: database.HealthStatusCorrupted, + Classification: &holes.Impact{Verdict: holes.VerdictDegraded, TotalMissing: 2, LongestRun: 1}} + update, _ := env.hw.prepareUpdateForResult(context.Background(), &fh, ev) + assert.Equal(t, database.UpdateTypeCorrupted, update.Type, "a degraded peer import is a failed one at once, no PAR2 repair queued") +} + +type fakeHeaderVerifier struct { + err error + calls []string +} + +func (v *fakeHeaderVerifier) VerifyPeerHeaders(_ context.Context, p string) error { + v.calls = append(v.calls, p) + return v.err +} + +func TestPeerBundle_HeaderSampleRunsOnlyForFullCheckFilesAndDecidesTheVerdict(t *testing.T) { + for name, tc := range map[string]struct { + err error + want EventType + }{ + "matches": {nil, EventTypeFileHealthy}, + "contradicts": {ErrPeerLayoutMismatch, EventTypeFileCorrupted}, + "inconclusive": {errors.New("provider down"), EventTypeCheckInconclusive}, + } { + t.Run(name, func(t *testing.T) { + env := sampledEnv(t) + v := &fakeHeaderVerifier{err: tc.err} + env.checker.SetPeerHeaderVerifier(v) + opts := CheckOptions{FullCheckFiles: map[string]struct{}{env.filePath: {}}} + ev := env.checker.CheckFile(context.Background(), env.filePath, opts) + require.Equal(t, tc.want, ev.Type) + assert.Equal(t, []string{env.filePath}, v.calls) + if tc.want == EventTypeFileCorrupted { + require.Error(t, ev.Error) + assert.ErrorIs(t, ev.Error, ErrPeerLayoutMismatch) + } + }) + } + t.Run("ordinary files never call it", func(t *testing.T) { + env := sampledEnv(t) + v := &fakeHeaderVerifier{err: ErrPeerLayoutMismatch} + env.checker.SetPeerHeaderVerifier(v) + ev := env.checker.CheckFile(context.Background(), env.filePath, CheckOptions{}) + assert.Equal(t, EventTypeFileHealthy, ev.Type) + assert.Empty(t, v.calls) + }) +} diff --git a/internal/health/worker.go b/internal/health/worker.go index 9bed82308..19558716b 100644 --- a/internal/health/worker.go +++ b/internal/health/worker.go @@ -604,6 +604,20 @@ func (hw *HealthWorker) prepareUpdateForResult(ctx context.Context, fh *database update.ErrorMessage = errorMsg update.ErrorDetails = event.Details + // A pending peer import never ends degraded: a release a peer vouched for + // that plays with missing segments is a failed one (the fast path then + // imports the NZB normally). It is written as corrupted so no degraded + // PAR2 repair is queued, and the gate is told (notifyPeerBundleResults). + if event.Classification != nil && event.Classification.Verdict == holes.VerdictDegraded && + fh.Status != database.HealthStatusRepairTriggered && hw.peerImportFailsFor(ctx, fh.FilePath) { + update.Type = database.UpdateTypeCorrupted + update.Status = database.HealthStatusCorrupted + return update, func() error { + slog.WarnContext(ctx, "Peer import is degraded; it is failed and removed", "file_path", fh.FilePath) + return nil + } + } + // Degraded verdict: the confirmed holes are within the padding caps, so // the file still plays (streaming zero-fills the gaps). Record it as // degraded — no repair trigger, no safety-folder move, periodic re-check — diff --git a/internal/importer/filesystem/safefoldername_test.go b/internal/importer/filesystem/safefoldername_test.go new file mode 100644 index 000000000..5bb23e741 --- /dev/null +++ b/internal/importer/filesystem/safefoldername_test.go @@ -0,0 +1,19 @@ +package filesystem + +import "testing" + +// The result of SafeFolderName is itself a safe folder name: applying the +// function again must not change it (a later NzbFolderPath pass strips the +// same extensions). +func TestSafeFolderNameIsAFixedPoint(t *testing.T) { + for _, in := range []string{"Movie.mkv.", "Movie.mkv ", "Movie.mkv.mkv.", "A.mp4. .", "x.ts.iso.", "Plain Title", "Dots...", " a b "} { + once, ok := SafeFolderName(in) + if !ok { + continue + } + twice, ok2 := SafeFolderName(once) + if !ok2 || twice != once { + t.Errorf("SafeFolderName(%q) = %q, again = %q (ok=%v)", in, once, twice, ok2) + } + } +} diff --git a/internal/importer/filesystem/utils.go b/internal/importer/filesystem/utils.go index dda939d08..99a4979f2 100644 --- a/internal/importer/filesystem/utils.go +++ b/internal/importer/filesystem/utils.go @@ -336,6 +336,24 @@ const maxSafeFolderNameLen = 200 // example the category folders) is refused. Unlike NzbFolderPath it can never // yield "." or "..". func SafeFolderName(title string, extraReserved ...string) (string, bool) { + // One pass can expose a new trailing extension (trimming "Movie.mkv." + // leaves "Movie.mkv"), so repeat until the name is a fixed point. Every + // pass that changes the name makes it strictly shorter, so this ends. + name, ok := safeFolderNameOnce(title, extraReserved) + for ok { + next, nok := safeFolderNameOnce(name, extraReserved) + if !nok { + return "", false + } + if next == name { + break + } + name = next + } + return name, ok +} + +func safeFolderNameOnce(title string, extraReserved []string) (string, bool) { name := strings.ToValidUTF8(title, "") if i := strings.LastIndexAny(name, `/\`); i >= 0 { name = name[i+1:] diff --git a/internal/importer/peer_bundle.go b/internal/importer/peer_bundle.go index fac262324..46ce13192 100644 --- a/internal/importer/peer_bundle.go +++ b/internal/importer/peer_bundle.go @@ -80,6 +80,15 @@ func (proc *Processor) SetPeerBundleFastPath(h PeerBundleFastPath) { proc.peerFastPath.Store(&peerFastPathHolder{h: h}) } +// SkipPeerFastPathOnce makes the next import of the queue item skip the peer +// fast path (a degraded peer import is being redone from its NZB). +func (proc *Processor) SkipPeerFastPathOnce(queueID int) { + proc.peerSkipOnce.Store(queueID, struct{}{}) +} + +// ClearSkipPeerFastPath undoes SkipPeerFastPathOnce. +func (proc *Processor) ClearSkipPeerFastPath(queueID int) { proc.peerSkipOnce.Delete(queueID) } + func (proc *Processor) setPeerItem(queueID int, info peerItemInfo) { proc.peerItems.Store(queueID, info) } @@ -101,6 +110,9 @@ func (proc *Processor) tryPeerBundle(ctx context.Context, n *nzbparser.Nzb, file result, written, taken = "", nil, false } }() + if _, skip := proc.peerSkipOnce.LoadAndDelete(queueID); skip { + return "", nil, false, nil + } // The common case: no peer offers anything. Nothing is hashed or built. if !holder.h.MayHaveOffers(ctx) { return "", nil, false, nil @@ -134,6 +146,13 @@ func (proc *Processor) tryPeerBundle(ctx context.Context, n *nzbparser.Nzb, file return out.ResultPath, out.WrittenPaths, true, nil } +// prebuiltStillDescribes says whether the store built for the peer lookup can +// be reused by the import's parse: only when the NZB was neither given gap +// placeholders nor had dead past-the-end articles trimmed meanwhile. +func prebuiltStillDescribes(p *prebuiltStore, gaps int, trimmed bool) bool { + return p != nil && gaps == 0 && !trimmed +} + // prebuiltStore is the NZB store (and message-id index) built for the peer // lookup, kept for the import's parse when the NZB was not changed meanwhile. type prebuiltStore struct { diff --git a/internal/importer/peer_bundle_prebuilt_test.go b/internal/importer/peer_bundle_prebuilt_test.go new file mode 100644 index 000000000..594b3c395 --- /dev/null +++ b/internal/importer/peer_bundle_prebuilt_test.go @@ -0,0 +1,42 @@ +package importer + +import ( + "context" + "testing" +) + +func TestPrebuiltStoreIsNotReusedAfterTrimOrGaps(t *testing.T) { + p := &prebuiltStore{} + if !prebuiltStillDescribes(p, 0, false) { + t.Fatal("an untouched NZB reuses the store") + } + if prebuiltStillDescribes(p, 0, true) { + t.Fatal("a trimmed NZB must rebuild the store") + } + if prebuiltStillDescribes(p, 2, false) { + t.Fatal("gap placeholders force a rebuild") + } + if prebuiltStillDescribes(nil, 0, false) { + t.Fatal("nothing to reuse") + } +} + +// A degraded peer import is redone from its NZB with the fast path skipped for +// that one run only. +func TestPeerFastPath_SkippedOnceForARequeuedItem(t *testing.T) { + e := newBatteryEnv(t) + fp := &fakeFastPath{take: true, env: e} + e.proc.SetPeerBundleFastPath(fp) + nzbPath := cleanNZB(e) + n := parseTestNzb(t, nzbPath) + + e.proc.SkipPeerFastPathOnce(5) + _, _, taken, _ := e.proc.tryPeerBundle(context.Background(), n, nzbPath, 5, "/", nil, nil, nil) + if taken || len(fp.reqs) != 0 { + t.Fatal("the fast path must not run for a requeued item") + } + _, _, taken, _ = e.proc.tryPeerBundle(context.Background(), n, nzbPath, 5, "/", nil, nil, nil) + if !taken { + t.Fatal("the skip applies once only") + } +} diff --git a/internal/importer/peer_bundle_service.go b/internal/importer/peer_bundle_service.go index 004bd2aa8..b410ccbe4 100644 --- a/internal/importer/peer_bundle_service.go +++ b/internal/importer/peer_bundle_service.go @@ -104,6 +104,33 @@ func (s *Service) FailImport(ctx context.Context, queueID int64, reason string) return nil } +// RequeueNormalImport puts the held queue item of a fast-path import back for +// a normal import (the peer fast path is off for it, once), after its peer +// files were removed. It reports false, changing nothing, for an item that has +// no NZB to import from (a direct peer import, or an NZB that is gone). +func (s *Service) RequeueNormalImport(ctx context.Context, queueID int64) (bool, error) { + item, err := s.database.Repository.GetQueueItem(ctx, queueID) + if err != nil { + return false, err + } + if item == nil || item.IsPeerImport() { + return false, nil + } + if _, err := os.Stat(item.NzbPath); err != nil { + return false, nil + } + s.processor.SkipPeerFastPathOnce(int(queueID)) + changed, err := s.bundleState().RequeueHeldQueueItem(ctx, queueID) + if err != nil || !changed { + s.processor.ClearSkipPeerFastPath(int(queueID)) + return false, err + } + if s.broadcaster != nil { + s.broadcaster.BroadcastQueueChanged() + } + return true, nil +} + // ReleaseHold lifts the hold of a queue item no peer file refers to any more. func (s *Service) ReleaseHold(ctx context.Context, queueID int64) error { if err := s.bundleState().SetQueueHold(ctx, queueID, false); err != nil { diff --git a/internal/importer/processor.go b/internal/importer/processor.go index c83efd1ad..395a95d77 100644 --- a/internal/importer/processor.go +++ b/internal/importer/processor.go @@ -60,6 +60,7 @@ type Processor struct { peerFastPath atomic.Pointer[peerFastPathHolder] // optional; nil unless the peer network is on pwRecorder atomic.Pointer[nzbPasswordHolder] // optional; nil unless the peer network is on peerItems sync.Map // queue id -> peerItemInfo + peerSkipOnce sync.Map // queue id -> {}: skip the peer fast path once // Pre-compiled regex patterns for RAR file sorting rarPartPattern *regexp.Regexp // pattern.part###.rar @@ -1078,7 +1079,9 @@ func (proc *Processor) ProcessNzbFile(ctx context.Context, filePath, relativePat // Trimmed here, not in the check: the warm-up reads n.Files until // warmDone closes. var pastEnd *articlesPastEndError + trimmed := false if errors.As(fastFailErr, &pastEnd) { + trimmed = true proc.parser.TrimArticles(ctx, n.Files, pastEnd.keep) brokenIdx, missingIDs, degradedFiles, fastFailErr = proc.preParseFastFail(ctx, n, cfg, queueID, category, downloadID) } @@ -1105,7 +1108,7 @@ func (proc *Processor) ProcessNzbFile(ctx context.Context, filePath, relativePat // The store built for the peer lookup describes this NZB exactly // unless gap placeholders were inserted into it (a nested NZB, parsed // afresh below, never reuses it). - if prebuilt != nil && gaps == 0 { + if prebuiltStillDescribes(prebuilt, gaps, trimmed) { parseOpts.PrebuiltStore, parseOpts.PrebuiltIndex = prebuilt.store, prebuilt.index } prebuilt = nil diff --git a/internal/peernet/bridge/bundleimport.go b/internal/peernet/bridge/bundleimport.go index 4669217c3..695fe59ce 100644 --- a/internal/peernet/bridge/bundleimport.go +++ b/internal/peernet/bridge/bundleimport.go @@ -11,6 +11,7 @@ import ( metapb "github.com/kipsilabs/altmount/internal/metadata/proto" "github.com/kipsilabs/altmount/internal/peernet/bundle" "github.com/kipsilabs/altmount/internal/peernet/bundleimport" + "github.com/kipsilabs/altmount/internal/peernet/bundlex" "github.com/kipsilabs/altmount/internal/peernet/contentid" "github.com/kipsilabs/altmount/internal/peernet/service" ) @@ -92,6 +93,16 @@ func encodedSizes(store *metapb.NzbStore) [][]int64 { return out } +// subjectNames returns the file name each NZB entry's subject carries ("" when +// it carries none), in NZB order. +func subjectNames(store *metapb.NzbStore) []string { + out := make([]string, 0, len(store.GetFiles())) + for _, f := range store.GetFiles() { + out = append(out, bundleimport.SubjectFilename(f.GetSubject())) + } + return out +} + func canonicalIDs(store *metapb.NzbStore) [][]string { out := make([][]string, 0, len(store.GetFiles())) for _, f := range store.GetFiles() { @@ -117,7 +128,17 @@ func (f *fastPath) TryBundle(ctx context.Context, req importer.PeerBundleRequest return importer.PeerBundleOutcome{}, false } fetchCtx, cancel := context.WithTimeout(ctx, f.cfg.FetchTimeout) - b, origin, err := f.cfg.Fetcher.FetchBundle(fetchCtx, rid.String()) + // The caller's requirements go to the fetcher, which filters the offers + // before dialing anyone: with the switch off an open-network offer is never + // contacted (a contact reveals the Release ID). + var b *bundle.Bundle + var origin bundleimport.Origin + if of, ok := f.cfg.Fetcher.(bundleimport.OptionFetcher); ok { + b, origin, err = of.FetchBundleWith(fetchCtx, rid.String(), bundleimport.FetchOptions{ + AllowOpen: f.cfg.OpenFastPath != nil && f.cfg.OpenFastPath(), MinOpenCorroborators: minOpenCorroborators}) + } else { + b, origin, err = f.cfg.Fetcher.FetchBundle(fetchCtx, rid.String()) + } cancel() if err != nil || b == nil || b.Scheme != bundle.SchemeNZB || b.ReleaseID != rid { return importer.PeerBundleOutcome{}, false @@ -128,7 +149,7 @@ func (f *fastPath) TryBundle(ctx context.Context, req importer.PeerBundleRequest acceptCtx, cancelAccept := context.WithTimeout(ctx, f.cfg.AcceptTimeout) defer cancelAccept() res, err := f.cfg.Acceptor.Accept(acceptCtx, bundleimport.Request{ - Bundle: b, Origin: origin, ExpectedNZB: canonicalIDs(req.NZB), ExpectedNZBSizes: encodedSizes(req.NZB), + Bundle: b, Origin: origin, ExpectedNZB: canonicalIDs(req.NZB), ExpectedNZBSizes: encodedSizes(req.NZB), ExpectedNZBNames: subjectNames(req.NZB), Target: bundleimport.Target{Dir: req.TargetDir, Category: req.Category, QueueID: int64(req.QueueID), TargetPath: req.TargetPath, SkipArr: req.SkipArr}, }) @@ -215,6 +236,16 @@ func (a serviceAdapter) ReleaseHold(ctx context.Context, queueID int64) error { return a.s.ReleaseHold(ctx, queueID) } +// RequeueForNormalImport implements bundleimport.Requeuer when the service can. +func (a serviceAdapter) RequeueForNormalImport(ctx context.Context, queueID int64) (bool, error) { + if rq, ok := a.s.(interface { + RequeueNormalImport(ctx context.Context, queueID int64) (bool, error) + }); ok { + return rq.RequeueNormalImport(ctx, queueID) + } + return false, nil +} + func (a serviceAdapter) RecordPeerImport(ctx context.Context, rec bundleimport.ImportRecord) (int64, error) { return a.s.RecordPeerImport(ctx, importer.PeerImportRecord{ Title: rec.Title, Category: rec.Category, TargetPath: rec.TargetPath, ResultingPath: rec.ResultingPath, @@ -243,11 +274,28 @@ func NewBundleFetcher(s BundleService) bundleimport.Fetcher { func (f bundleFetcher) FetchBundle(ctx context.Context, releaseID string) (*bundle.Bundle, bundleimport.Origin, error) { b, src, err := f.s.FetchBundle(ctx, releaseID) + return originOf(b, src, err) +} + +// FetchBundleWith filters the offers by the caller's requirements before any +// peer is contacted, when the service can. +func (f bundleFetcher) FetchBundleWith(ctx context.Context, releaseID string, o bundleimport.FetchOptions) (*bundle.Bundle, bundleimport.Origin, error) { + ws, ok := f.s.(interface { + FetchBundleWith(ctx context.Context, releaseID string, opts bundlex.FetchOptions) (*bundle.Bundle, service.FetchSource, error) + }) + if !ok { + return f.FetchBundle(ctx, releaseID) + } + b, src, err := ws.FetchBundleWith(ctx, releaseID, bundlex.FetchOptions{Restrict: true, AllowOpen: o.AllowOpen, MinOpenCorroborators: o.MinOpenCorroborators}) + return originOf(b, src, err) +} + +func originOf(b *bundle.Bundle, src service.FetchSource, err error) (*bundle.Bundle, bundleimport.Origin, error) { if err != nil { return nil, bundleimport.Origin{}, err } return b, bundleimport.Origin{NetworkID: src.Network.String(), PublisherID: src.Publisher.String(), - Open: src.Network.IsOpen(), Corroborators: src.Corroborators}, nil + Open: src.Network.IsOpen(), Corroborators: src.Corroborators, BundleID: src.BundleID.String()}, nil } type bundleReporter struct{ s BundleService } @@ -265,6 +313,16 @@ func (r bundleReporter) ReportBadBundle(_ context.Context, o bundleimport.Origin r.s.NoteBundleFullCheck(o.NetworkID, o.PublisherID, false) } +// ReportFailedBundle puts a layout that failed local verification on the +// negative cache when the service keeps one. +func (r bundleReporter) ReportFailedBundle(_ context.Context, bundleID, releaseID string) { + if nf, ok := r.s.(interface { + NoteBundleFailed(bundleID, releaseID string) + }); ok { + nf.NoteBundleFailed(bundleID, releaseID) + } +} + func (r bundleReporter) ReportVerifiedBundle(_ context.Context, o bundleimport.Origin, _ string) { r.s.NoteBundleFullCheck(o.NetworkID, o.PublisherID, true) } diff --git a/internal/peernet/bridge/bundleimport_test.go b/internal/peernet/bridge/bundleimport_test.go index 470082d08..6d81aafbd 100644 --- a/internal/peernet/bridge/bundleimport_test.go +++ b/internal/peernet/bridge/bundleimport_test.go @@ -168,7 +168,7 @@ func TestBundleFetcherAdapterMapsSourceToOrigin(t *testing.T) { b, o, err := NewBundleFetcher(svc).FetchBundle(context.Background(), "rel") require.NoError(t, err) assert.Same(t, svc.b, b) - assert.Equal(t, bundleimport.Origin{NetworkID: nw.String(), PublisherID: pub.String(), Open: true}, o) + assert.Equal(t, bundleimport.Origin{NetworkID: nw.String(), PublisherID: pub.String(), Open: true, BundleID: strings.Repeat("0", 64)}, o) assert.Equal(t, "rel", svc.gotRel) svc.err = errors.New("no offers") _, _, err = NewBundleFetcher(svc).FetchBundle(context.Background(), "rel") @@ -234,3 +234,34 @@ func TestFastPath_VerificationThatDoesNotFitDeclinesInsteadOfStallingTheImport(t assert.False(t, ok) assert.Less(t, time.Since(start), 2*time.Second) } + +// optionFetcher records the requirements it is asked to apply. +type optionFetcher struct { + originFetcher + opts []bundleimport.FetchOptions + plain atomic.Int32 +} + +func (f *optionFetcher) FetchBundle(ctx context.Context, id string) (*bundle.Bundle, bundleimport.Origin, error) { + f.plain.Add(1) + return f.originFetcher.FetchBundle(ctx, id) +} + +func (f *optionFetcher) FetchBundleWith(ctx context.Context, id string, o bundleimport.FetchOptions) (*bundle.Bundle, bundleimport.Origin, error) { + f.opts = append(f.opts, o) + return f.originFetcher.FetchBundle(ctx, id) +} + +// The switch and the corroboration floor are handed to the fetcher so offers +// are filtered before any peer is dialled (a contact reveals the Release ID). +func TestFastPath_RequirementsGoToTheFetcherBeforeAnyDial(t *testing.T) { + for _, on := range []bool{false, true} { + f := &optionFetcher{originFetcher: originFetcher{b: matching(t), o: bundleimport.Origin{NetworkID: "p"}}} + h := NewPeerBundleFastPath(FastPathConfig{Acceptor: &fakeAcceptor{res: &bundleimport.Result{Dir: "d", Paths: []string{"d/a.mkv"}}}, Fetcher: f, + Enabled: func() bool { return true }, OpenFastPath: func() bool { return on }}) + _, _ = h.TryBundle(context.Background(), req()) + assert.Zero(t, f.plain.Load(), "the unfiltered fetch is never used when the fetcher can filter") + require.Len(t, f.opts, 1) + assert.Equal(t, bundleimport.FetchOptions{AllowOpen: on, MinOpenCorroborators: minOpenCorroborators}, f.opts[0]) + } +} diff --git a/internal/peernet/bundle/budget_test.go b/internal/peernet/bundle/budget_test.go index e08c74241..d1aef2bad 100644 --- a/internal/peernet/bundle/budget_test.go +++ b/internal/peernet/bundle/budget_test.go @@ -47,7 +47,7 @@ func repeat32(v byte) []byte { } func TestHostileClipsCannotBloatVerify(t *testing.T) { - raw := hostileClipBundle(t, 200) + raw := hostileClipBundle(t, 50) if len(raw) > MaxDecompressedSize { t.Fatalf("test input is %d bytes canonical", len(raw)) } @@ -63,7 +63,7 @@ func TestHostileClipsCannotBloatVerify(t *testing.T) { var before, after runtime.MemStats runtime.GC() runtime.ReadMemStats(&before) - _, err = VerifyCanonical(raw, id, [32]byte{0xAA}, Options{}) + _, err = VerifyCanonical(raw, id, [32]byte{0xAA}) runtime.ReadMemStats(&after) if err == nil { t.Fatal("hostile bundle verified") diff --git a/internal/peernet/bundle/compress.go b/internal/peernet/bundle/compress.go index 4be046569..f1a61879b 100644 --- a/internal/peernet/bundle/compress.go +++ b/internal/peernet/bundle/compress.go @@ -60,7 +60,7 @@ func Decompress(wire []byte) ([]byte, error) { return nil, fmt.Errorf("%w: zstd: %v", ErrMalformed, err) } defer dec.Close() - out, err := readBounded(dec) + out, err := readBounded(dec, declaredSize(wire)) switch { case len(out) > MaxDecompressedSize: return nil, fmt.Errorf("%w: more than %d bytes uncompressed", ErrTooLarge, MaxDecompressedSize) @@ -77,7 +77,17 @@ func Decompress(wire []byte) ([]byte, error) { // readBounded reads r in fixed chunks, up to MaxDecompressedSize+1 bytes, and // joins them once, so peak memory stays near twice the output rather than the // several times that a growing buffer costs. -func readBounded(r io.Reader) ([]byte, error) { +func readBounded(r io.Reader, declared int) ([]byte, error) { + if declared > 0 { + // The frame states its content size: read into one exact buffer (one + // byte of slack detects a frame that lies and keeps going). + out := make([]byte, declared+1) + n, err := io.ReadFull(r, out) + if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { + err = nil + } + return out[:n], err + } const chunk = 1 << 20 var chunks [][]byte total := 0 @@ -106,3 +116,14 @@ func readBounded(r io.Reader) ([]byte, error) { } return out, err } + +// declaredSize returns the content size a frame header states, or 0 when it +// states none or more than MaxDecompressedSize (the streaming path then +// enforces the bound). +func declaredSize(wire []byte) int { + var h zstd.Header + if err := h.Decode(wire); err != nil || !h.HasFCS || h.FrameContentSize == 0 || h.FrameContentSize > MaxDecompressedSize { + return 0 + } + return int(h.FrameContentSize) +} diff --git a/internal/peernet/bundle/compress_test.go b/internal/peernet/bundle/compress_test.go index 9a4352e4b..8279cfc73 100644 --- a/internal/peernet/bundle/compress_test.go +++ b/internal/peernet/bundle/compress_test.go @@ -61,7 +61,7 @@ func TestDecompressRejects(t *testing.T) { {"garbage", []byte("this is not zstd at all, just text"), ErrMalformed}, {"truncated frame", good[:len(good)-2], ErrMalformed}, {"input over 8 MB", make([]byte, MaxCompressedSize+1), ErrTooLarge}, - {"bomb over 64 MB", zeroBomb(t, MaxDecompressedSize+1, 0), ErrTooLarge}, + {"bomb over the cap", zeroBomb(t, MaxDecompressedSize+1, 0), ErrTooLarge}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/peernet/bundle/fuzz_test.go b/internal/peernet/bundle/fuzz_test.go index 5f95651bc..e0ee21423 100644 --- a/internal/peernet/bundle/fuzz_test.go +++ b/internal/peernet/bundle/fuzz_test.go @@ -138,6 +138,9 @@ func FuzzValidate(f *testing.F) { f.Add([]byte{}, uint8(3), int64(-1)) f.Add([]byte{}, uint8(36), int64(1<<62)) f.Add([]byte{}, uint8(70), int64(576)) + // The good bundles are built (and their ids hashed) once; every exec + // mutates a cheap deep copy, so the fuzzer spends its time on Validate. + bases := []*Bundle{goodBundle(f), goodNZBBundle(f)} f.Fuzz(func(t *testing.T, data []byte, sel uint8, v int64) { check := func(b *Bundle) { if err := Validate(b); err != nil { @@ -158,8 +161,8 @@ func FuzzValidate(f *testing.F) { if b, err := decodeRaw(data); err == nil { check(b) } - for _, mk := range []func() *Bundle{func() *Bundle { return goodBundle(t) }, func() *Bundle { return goodNZBBundle(t) }} { - b := mk() + for _, base := range bases { + b := clone(base) mutateBundle(b, sel, v) check(b) } diff --git a/internal/peernet/bundle/heap_test.go b/internal/peernet/bundle/heap_test.go new file mode 100644 index 000000000..5d968042b --- /dev/null +++ b/internal/peernet/bundle/heap_test.go @@ -0,0 +1,92 @@ +package bundle + +import ( + "fmt" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +// heapWorstBundle is the largest-pool shape a bundle can take: 60k pool +// entries of ~220 bytes, one NZB-scheme file reading all of them. +func heapWorstBundle(t *testing.T) (wire []byte, id BundleID, rel contentid.ID, canonical int) { + t.Helper() + n := 60000 + pad := strings.Repeat("a", 200) + pool := make([]string, n) + segs := make([]Segment, n) + for i := range pool { + pool[i] = fmt.Sprintf("%s%08d@h.example", pad, i) + segs[i] = Segment{Ref: int64(i), Size: 1, Start: 0, End: 0} + } + b := &Bundle{Scheme: SchemeNZB, Title: "T", PostedUnix: 1700000000, Pool: pool, ReleaseFiles: []int{n}, + Files: []File{{Name: "m.mkv", Size: int64(n), Segments: segs, FileID: contentid.ID{1}}}, TotalSize: int64(n), ReleaseID: contentid.ID{9}} + fid, err := contentid.FileID(buildMeta(b, &b.Files[0])) + if err != nil { + t.Fatal(err) + } + b.Files[0].FileID = fid + if b.ReleaseID, err = releaseIDOfList(b.Pool, b.ReleaseFiles); err != nil { + t.Fatal(err) + } + raw, err := Encode(b) + if err != nil { + t.Fatal(err) + } + wire, err = Compress(raw) + if err != nil { + t.Fatal(err) + } + return wire, ComputeID(raw), b.ReleaseID, len(raw) +} + +// TestVerifyPeakHeap pins the memory a single Verify of a worst-case valid +// bundle may use, so the decode path cannot silently regain extra copies. +func TestVerifyPeakHeap(t *testing.T) { + if testing.Short() { + t.Skip("allocates hundreds of MB") + } + wire, id, rel, canonical := heapWorstBundle(t) + runtime.GC() + var base runtime.MemStats + runtime.ReadMemStats(&base) + var ( + mu sync.Mutex + peak uint64 + ) + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + var m runtime.MemStats + for { + select { + case <-stop: + return + default: + } + runtime.ReadMemStats(&m) + mu.Lock() + peak = max(peak, m.HeapAlloc) + mu.Unlock() + time.Sleep(time.Millisecond) + } + }() + if _, err := Verify(wire, id, rel); err != nil { + t.Fatal(err) + } + close(stop) + <-done + extra := int64(peak) - int64(base.HeapAlloc) + t.Logf("canonical %d MB, wire %d KB, peak extra heap %d MB (%.1fx canonical)", canonical>>20, len(wire)>>10, extra>>20, float64(extra)/float64(canonical)) + if limit := int64(canonical) * maxHeapFactorTenths / 10; extra > limit && !raceBuild { + t.Fatalf("peak extra heap %d MB exceeds %.1fx canonical (%d MB)", extra>>20, float64(maxHeapFactorTenths)/10, limit>>20) + } +} + +// maxHeapFactorTenths is the allowed peak, in tenths of the canonical size. +const maxHeapFactorTenths = 55 diff --git a/internal/peernet/bundle/layout_binding_test.go b/internal/peernet/bundle/layout_binding_test.go new file mode 100644 index 000000000..94d147b8a --- /dev/null +++ b/internal/peernet/bundle/layout_binding_test.go @@ -0,0 +1,102 @@ +package bundle + +import ( + "errors" + "fmt" + "testing" +) + +func mkSeg(r int64, size int64) Segment { return Segment{Ref: r, Size: size, Start: 0, End: size - 1} } + +// filesBundle wraps hand-made files in a files-scheme bundle that Validate +// accepts apart from the property a test exercises. +func filesBundle(pool []string, files ...File) *Bundle { + b := &Bundle{Scheme: SchemeFiles, Title: "T", PostedUnix: 1700000000, Pool: pool, ReleaseID: [32]byte{9}} + for i := range files { + files[i].FileID = [32]byte{byte(i + 1)} + b.TotalSize += files[i].Size + } + b.Files = files + return b +} + +func poolOf(n int) []string { + out := make([]string, n) + for i := range out { + out[i] = fmt.Sprintf("part%d@poc.example", i) + } + return out +} + +// P1: one article referenced 1,999 times must not pass as a 1.4 GB file. +func TestFilesSchemeRepeatedRefRejected(t *testing.T) { + segs := make([]Segment, 2000) + for i := range segs { + segs[i] = mkSeg(int64(min(i, 1)), 700000) + } + b := filesBundle(poolOf(2), File{Name: "Movie.mkv", Size: 2000 * 700000, Segments: segs}) + if err := Validate(b); !errors.Is(err, ErrCoverage) { + t.Fatalf("want ErrCoverage, got %v", err) + } +} + +// P2: interior segments that skip bytes must not shrink a big release. +func TestInteriorPartialRangeRejected(t *testing.T) { + segs := make([]Segment, 100) + for i := range segs { + segs[i] = Segment{Ref: int64(i), Size: 700000, Start: 350000, End: 350009} + } + b := filesBundle(poolOf(100), File{Name: "Movie.mkv", Size: 1000, Segments: segs}) + if err := Validate(b); !errors.Is(err, ErrPartialRange) { + t.Fatalf("want ErrPartialRange, got %v", err) + } +} + +func TestPartialEdgesStillAccepted(t *testing.T) { + segs := []Segment{ + {Ref: 0, Size: 1000, Start: 100, End: 999}, + mkSeg(1, 1000), + {Ref: 2, Size: 1000, Start: 0, End: 499}, + } + b := filesBundle(poolOf(3), File{Name: "Movie.mkv", Size: 900 + 1000 + 500, Segments: segs}) + if err := Validate(b); err != nil { + t.Fatal(err) + } + one := filesBundle(poolOf(1), File{Name: "a.bin", Size: 10, Segments: []Segment{{Ref: 0, Size: 1000, Start: 5, End: 14}}}) + if err := Validate(one); err != nil { + t.Fatalf("single segment with both edges partial: %v", err) + } +} + +// P4: two files sharing their first article must not both pass. +func TestFilesSchemeSharedFirstEntryRejected(t *testing.T) { + a := File{Name: "A.mkv", Size: 3000, Segments: []Segment{mkSeg(0, 1000), mkSeg(1, 1000), mkSeg(2, 1000)}} + c := File{Name: "B.mkv", Size: 3000, Segments: []Segment{mkSeg(0, 1000), mkSeg(3, 1000), mkSeg(4, 1000)}} + if err := Validate(filesBundle(poolOf(5), a, c)); !errors.Is(err, ErrCoverage) { + t.Fatalf("want ErrCoverage, got %v", err) + } +} + +// An identical table (shared outer volume, PAR2 listed by two files) stays legal. +func TestIdenticalSharedTableAccepted(t *testing.T) { + par := Par2File{Name: "x.par2", Size: 1000, Segments: []Segment{mkSeg(2, 1000)}} + a := File{Name: "A.mkv", Size: 1000, Segments: []Segment{mkSeg(0, 1000)}, Par2: []Par2File{par}} + c := File{Name: "B.mkv", Size: 1000, Segments: []Segment{mkSeg(1, 1000)}, Par2: []Par2File{par}} + if err := Validate(filesBundle(poolOf(3), a, c)); err != nil { + t.Fatal(err) + } + // Same articles but a different size claim is not the identical table. + par2 := Par2File{Name: "x.par2", Size: 900, Segments: []Segment{{Ref: 2, Size: 1000, Start: 0, End: 899}}} + c.Par2 = []Par2File{par2} + if err := Validate(filesBundle(poolOf(3), a, c)); !errors.Is(err, ErrCoverage) { + t.Fatalf("want ErrCoverage, got %v", err) + } +} + +func TestFilesSchemeSegmentCeiling(t *testing.T) { + big := int64(MaxFilesSchemeSegment + 1) + b := filesBundle(poolOf(1), File{Name: "a.bin", Size: big, Segments: []Segment{mkSeg(0, big)}}) + if err := Validate(b); !errors.Is(err, ErrImplausible) { + t.Fatalf("want ErrImplausible, got %v", err) + } +} diff --git a/internal/peernet/bundle/limits.go b/internal/peernet/bundle/limits.go index 873771dc1..07c20696c 100644 --- a/internal/peernet/bundle/limits.go +++ b/internal/peernet/bundle/limits.go @@ -3,7 +3,7 @@ package bundle import "errors" // Limits. Several are design calls the owner may tune; the hard caps named in -// the design (8 MB compressed, 64 MB decompressed, 500k segments, 10k files) +// the design (8 MB compressed, 16 MB decompressed, 500k segments, 10k files) // are enforced while streaming and again on the decoded structure. const ( // SchemaVersion is the only major version this package reads and writes. @@ -13,7 +13,7 @@ const ( MaxCompressedSize = 8 << 20 // MaxDecompressedSize bounds the canonical (uncompressed) bytes. The // decompressor stops as soon as the output would exceed it. - MaxDecompressedSize = 64 << 20 + MaxDecompressedSize = 16 << 20 // MaxWindowSize is the largest zstd window a frame may ask for. The // builder uses the same value, so honest bundles always fit. MaxWindowSize = 8 << 20 @@ -58,6 +58,11 @@ const ( // MaxSegmentSize is the largest decoded size of one article: real posts // are 0.25-1 MB, a few go to a few MB. MaxSegmentSize = 8 << 20 + // MaxFilesSchemeSegment bounds a declared segment size in the "files" + // scheme, where no NZB confirms it. Posters use 0.25-1 MB articles + // (the importer's own estimates assume ~768 KB) and a few go to a few MB; + // nothing in the importer or the readers expects more than 4 MiB. + MaxFilesSchemeSegment = 4 << 20 // MaxFileSize caps one file and the bundle total (1 PiB, as records). MaxFileSize = int64(1) << 50 // MaxClipDelta bounds a clip's 90 kHz offset (about 10 years). @@ -96,5 +101,6 @@ var ( ErrNotShareable = errors.New("bundle: release is not shareable") ErrUnresolved = errors.New("bundle: metadata segments are not resolved") ErrCoverage = errors.New("bundle: segments are not exactly one NZB file entry") + ErrPartialRange = errors.New("bundle: an interior segment skips bytes") ErrSegmentNotInNZB = errors.New("bundle: file segment is not in the release segment list") ) diff --git a/internal/peernet/bundle/race_off_test.go b/internal/peernet/bundle/race_off_test.go new file mode 100644 index 000000000..7bd3c70e4 --- /dev/null +++ b/internal/peernet/bundle/race_off_test.go @@ -0,0 +1,5 @@ +//go:build !race + +package bundle + +const raceBuild = false diff --git a/internal/peernet/bundle/race_on_test.go b/internal/peernet/bundle/race_on_test.go new file mode 100644 index 000000000..5d637fc4e --- /dev/null +++ b/internal/peernet/bundle/race_on_test.go @@ -0,0 +1,6 @@ +//go:build race + +package bundle + +// raceBuild: the race detector inflates the heap, so memory bounds are only logged. +const raceBuild = true diff --git a/internal/peernet/bundle/validate.go b/internal/peernet/bundle/validate.go index 615d11932..072b1c896 100644 --- a/internal/peernet/bundle/validate.go +++ b/internal/peernet/bundle/validate.go @@ -1,6 +1,8 @@ package bundle import ( + "crypto/sha256" + "encoding/binary" "fmt" "sort" "strings" @@ -68,8 +70,13 @@ type validator struct { b *Bundle segs int extents int - used []bool // pool entries referenced (files scheme) - names map[string]struct{} + // owner maps a pool entry to the segment table that references it (-1 = + // none). A pool entry belongs to exactly one table; two usages of one table + // (a shared outer volume, a PAR2 file listed by several files) are the same + // table only when every reference, size and range is identical. + owner []int32 + tables map[TableKey]int32 + names map[string]struct{} // nzb scheme: the pool range of every NZB file entry, and which entries // are read inline (once, by one file) or as shared outer/PAR2 sources. entStart []int @@ -103,7 +110,7 @@ func Validate(b *Bundle) error { if err := checkPassword(b.NZBPassword); err != nil { return err } - v := &validator{b: b, names: map[string]struct{}{}} + v := &validator{b: b, names: map[string]struct{}{}, tables: map[TableKey]int32{}} if err := v.checkPool(); err != nil { return err } @@ -119,8 +126,8 @@ func Validate(b *Bundle) error { } } if b.Scheme == SchemeFiles { - for i, used := range v.used { - if !used { + for i, o := range v.owner { + if o < 0 { return fmt.Errorf("%w: pool entry %d is not referenced", ErrBadPool, i) } } @@ -181,6 +188,7 @@ func (v *validator) checkPool() error { pos += n } v.inline, v.sharedE = map[int]struct{}{}, map[int]struct{}{} + v.initOwner() return nil } if len(b.ReleaseFiles) != 0 { @@ -193,10 +201,17 @@ func (v *validator) checkPool() error { } seen[id] = struct{}{} } - v.used = make([]bool, len(b.Pool)) + v.initOwner() return nil } +func (v *validator) initOwner() { + v.owner = make([]int32, len(v.b.Pool)) + for i := range v.owner { + v.owner[i] = -1 + } +} + func validDate(d int64) bool { return d == 0 || (d >= 1 && d <= maxUnixSeconds) } func (v *validator) checkFile(f *File) error { @@ -326,12 +341,43 @@ func (v *validator) checkInline(f *File) error { return nil } +// TableKey identifies a segment table by its whole content: every reference, +// size and usable range, in order. +type TableKey [32]byte + +// KeyOf hashes a segment table (references, sizes and ranges, in order). +func KeyOf(segs []Segment) TableKey { + h := sha256.New() + var buf [32]byte + for _, s := range segs { + binary.BigEndian.PutUint64(buf[0:], uint64(s.Ref)) + binary.BigEndian.PutUint64(buf[8:], uint64(s.Size)) + binary.BigEndian.PutUint64(buf[16:], uint64(s.Start)) + binary.BigEndian.PutUint64(buf[24:], uint64(s.End)) + h.Write(buf[:]) + } + var k TableKey + h.Sum(k[:0]) + return k +} + // checkTable validates one segment table and returns the bytes it covers. +// +// A table reads one posted file front to back. Only its first segment may +// start inside the article and only its last may end inside it (a single +// segment may do both); every other segment is used whole, so no usable range +// can hide bytes in the middle. A pool entry is referenced at most once within +// a table, and by one table only unless the other usage is identical. func (v *validator) checkTable(segs []Segment) (int64, error) { if v.segs += len(segs); v.segs > MaxSegments { return 0, fmt.Errorf("%w: more than %d segments", ErrLimit, MaxSegments) } poolLen := int64(len(v.b.Pool)) + ceiling := int64(MaxSegmentSize) + if v.b.Scheme == SchemeFiles { + ceiling = MaxFilesSchemeSegment + } + seen := make(map[int64]struct{}, len(segs)) var covered int64 for i, s := range segs { switch { @@ -340,18 +386,33 @@ func (v *validator) checkTable(segs []Segment) (int64, error) { return 0, fmt.Errorf("%w: segment %d is a gap", ErrInvalidField, i) case s.Ref < 0 || s.Ref >= poolLen: return 0, fmt.Errorf("%w: segment %d references pool entry %d of %d", ErrOutOfRange, i, s.Ref, poolLen) - case s.Size > MaxSegmentSize: + case s.Size > ceiling: return 0, fmt.Errorf("%w: segment %d is %d bytes", ErrImplausible, i, s.Size) case s.Size < 1: return 0, fmt.Errorf("%w: segment %d size %d", ErrOutOfRange, i, s.Size) case s.Start < 0 || s.End < s.Start || s.End > s.Size-1: return 0, fmt.Errorf("%w: segment %d range [%d,%d] of %d", ErrOutOfRange, i, s.Start, s.End, s.Size) + case i > 0 && s.Start != 0, i < len(segs)-1 && s.End != s.Size-1: + return 0, fmt.Errorf("%w: segment %d range [%d,%d] of %d", ErrPartialRange, i, s.Start, s.End, s.Size) } - if s.Ref >= 0 && v.used != nil { - v.used[s.Ref] = true + if _, dup := seen[s.Ref]; dup { + return 0, fmt.Errorf("%w: segment %d repeats pool entry %d within one table", ErrCoverage, i, s.Ref) } + seen[s.Ref] = struct{}{} covered += s.End - s.Start + 1 } + key := KeyOf(segs) + if _, same := v.tables[key]; same { + return covered, nil + } + id := int32(len(v.tables)) + v.tables[key] = id + for i, s := range segs { + if v.owner[s.Ref] >= 0 { + return 0, fmt.Errorf("%w: segment %d shares pool entry %d with a different table", ErrCoverage, i, s.Ref) + } + v.owner[s.Ref] = id + } return covered, nil } diff --git a/internal/peernet/bundle/validate_test.go b/internal/peernet/bundle/validate_test.go index bfd2178c5..e563a3e94 100644 --- a/internal/peernet/bundle/validate_test.go +++ b/internal/peernet/bundle/validate_test.go @@ -44,7 +44,7 @@ func TestValidateRejects(t *testing.T) { s := &fileNamed(t, b, "movie.mkv").Segments[0] s.Size, s.End = math.MaxInt64, math.MaxInt64-1 }, ErrImplausible}, - {"one byte gap inside the extents", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Segments[1].Start = 1 }, ErrSizeMismatch}, + {"one byte gap inside the extents", false, func(b *Bundle) { fileNamed(t, b, "movie.mkv").Segments[1].Start = 1 }, ErrPartialRange}, {"declared size far beyond the segments", false, func(b *Bundle) { f := fileNamed(t, b, "movie.mkv") b.TotalSize += (1 << 40) - f.Size diff --git a/internal/peernet/bundle/verify.go b/internal/peernet/bundle/verify.go index 8865582ab..796cc9ffc 100644 --- a/internal/peernet/bundle/verify.go +++ b/internal/peernet/bundle/verify.go @@ -1,20 +1,12 @@ package bundle import ( + "context" "fmt" "github.com/kipsilabs/altmount/internal/peernet/contentid" ) -// Options lets the caller replace the files-scheme Release ID function, so -// this package does not depend on the one in contentid until it exists. -type Options struct { - // FilesReleaseID computes the Release ID of the files scheme from the File - // IDs, which are passed sorted ascending bytewise. Nil selects - // FilesReleaseID. - FilesReleaseID func(sortedFileIDs []contentid.ID) contentid.ID -} - // FilesReleaseID is the files-scheme Release ID, as defined by // contentid.ReleaseIDFromFiles: SHA-256 over the "altmount-peernet/release-files/v1" // domain label and the File IDs sorted ascending bytewise. ids is not @@ -48,20 +40,38 @@ func Pack(b *Bundle) (wire []byte, id BundleID, err error) { // segments; the Release ID recomputed for the declared scheme equals both the // bundle's own and the announced one. It does not touch the network. func Verify(wire []byte, announcedBundle BundleID, announcedRelease contentid.ID) (*Bundle, error) { - return VerifyWith(wire, announcedBundle, announcedRelease, Options{}) + return VerifyContext(context.Background(), wire, announcedBundle, announcedRelease) } -// VerifyWith is Verify with options. -func VerifyWith(wire []byte, announcedBundle BundleID, announcedRelease contentid.ID, o Options) (*Bundle, error) { +// verifySlot admits one verification at a time, node-wide: decoding a large +// bundle peaks at several times its canonical size, so concurrent fetches, fast +// paths and playback requests must queue. +var verifySlot = make(chan struct{}, 1) + +// verifyEnterHook is a test seam; it returns the function to call on leaving. +var verifyEnterHook func() func() + +// VerifyContext is Verify that waits for the node-wide verification slot and +// gives up when ctx ends first. +func VerifyContext(ctx context.Context, wire []byte, announcedBundle BundleID, announcedRelease contentid.ID) (*Bundle, error) { + select { + case verifySlot <- struct{}{}: + defer func() { <-verifySlot }() + case <-ctx.Done(): + return nil, ctx.Err() + } + if verifyEnterHook != nil { + defer verifyEnterHook()() + } raw, err := Decompress(wire) if err != nil { return nil, err } - return VerifyCanonical(raw, announcedBundle, announcedRelease, o) + return VerifyCanonical(raw, announcedBundle, announcedRelease) } // VerifyCanonical is Verify for bytes that are already decompressed. -func VerifyCanonical(canonical []byte, announcedBundle BundleID, announcedRelease contentid.ID, o Options) (*Bundle, error) { +func VerifyCanonical(canonical []byte, announcedBundle BundleID, announcedRelease contentid.ID) (*Bundle, error) { if len(canonical) > MaxDecompressedSize { return nil, fmt.Errorf("%w: %d bytes uncompressed", ErrTooLarge, len(canonical)) } @@ -91,11 +101,7 @@ func VerifyCanonical(canonical []byte, announcedBundle BundleID, announcedReleas return nil, fmt.Errorf("%w: %v", ErrReleaseIDMismatch, err) } default: - fn := o.FilesReleaseID - if fn == nil { - fn = FilesReleaseID - } - got = fn(ids) + got = FilesReleaseID(ids) } if got != b.ReleaseID || got != announcedRelease { return nil, ErrReleaseIDMismatch diff --git a/internal/peernet/bundle/verify_test.go b/internal/peernet/bundle/verify_test.go index 96b52f233..987de8ba1 100644 --- a/internal/peernet/bundle/verify_test.go +++ b/internal/peernet/bundle/verify_test.go @@ -2,8 +2,12 @@ package bundle import ( "bytes" + "context" "errors" + "sync" + "sync/atomic" "testing" + "time" "github.com/kipsilabs/altmount/internal/peernet/contentid" ) @@ -156,32 +160,6 @@ func rebuildPoolDropping(t testing.TB, b *Bundle) *Bundle { return n } -func TestVerifyWithCustomFilesReleaseID(t *testing.T) { - b := goodBundle(t) - custom := contentid.ID{7, 7, 7} - b.ReleaseID = custom - wire, id := pack(t, b) - called := 0 - opts := Options{FilesReleaseID: func(ids []contentid.ID) contentid.ID { - called++ - if len(ids) != len(b.Files) { - t.Errorf("got %d ids", len(ids)) - } - for i := 1; i < len(ids); i++ { - if bytes.Compare(ids[i-1][:], ids[i][:]) >= 0 { - t.Error("ids not sorted ascending") - } - } - return custom - }} - if _, err := VerifyWith(wire, id, custom, opts); err != nil || called != 1 { - t.Fatalf("err %v called %d", err, called) - } - if _, err := Verify(wire, id, custom); !errors.Is(err, ErrReleaseIDMismatch) { - t.Fatalf("default function must not accept the custom id: %v", err) - } -} - func TestFilesReleaseID(t *testing.T) { ids := []contentid.ID{{3}, {1}, {2}} want := testFilesRelease(ids) @@ -199,10 +177,10 @@ func TestFilesReleaseID(t *testing.T) { func TestVerifyCanonical(t *testing.T) { b := goodNZBBundle(t) raw := mustEncode(t, b) - if _, err := VerifyCanonical(raw, ComputeID(raw), b.ReleaseID, Options{}); err != nil { + if _, err := VerifyCanonical(raw, ComputeID(raw), b.ReleaseID); err != nil { t.Fatal(err) } - if _, err := VerifyCanonical(raw, BundleID{}, b.ReleaseID, Options{}); !errors.Is(err, ErrBundleIDMismatch) { + if _, err := VerifyCanonical(raw, BundleID{}, b.ReleaseID); !errors.Is(err, ErrBundleIDMismatch) { t.Fatal(err) } } @@ -218,3 +196,49 @@ func TestVerifyNZBSchemeRecomputesTheFullList(t *testing.T) { t.Fatalf("got %v", err) } } + +// Verifying decodes tens of MB at its peak, so only one bundle is verified at a +// time node-wide; waiters respect their context. +func TestVerifyIsSerialisedNodeWide(t *testing.T) { + b := goodBundle(t) + wire, id := pack(t, b) + var cur, peak atomic.Int32 + verifyEnterHook = func() func() { + n := cur.Add(1) + for { + p := peak.Load() + if n <= p || peak.CompareAndSwap(p, n) { + break + } + } + time.Sleep(20 * time.Millisecond) + return func() { cur.Add(-1) } + } + t.Cleanup(func() { verifyEnterHook = nil }) + var wg sync.WaitGroup + for range 6 { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := Verify(wire, id, b.ReleaseID); err != nil { + t.Error(err) + } + }() + } + wg.Wait() + if peak.Load() != 1 { + t.Fatalf("%d bundles were verified at once, want 1", peak.Load()) + } +} + +func TestVerifyWaitersRespectTheirContext(t *testing.T) { + b := goodBundle(t) + wire, id := pack(t, b) + verifySlot <- struct{}{} // another verification is running + defer func() { <-verifySlot }() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + if _, err := VerifyContext(ctx, wire, id, b.ReleaseID); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("want the context error, got %v", err) + } +} diff --git a/internal/peernet/bundle/wire.go b/internal/peernet/bundle/wire.go index de61f0616..ee2b4889f 100644 --- a/internal/peernet/bundle/wire.go +++ b/internal/peernet/bundle/wire.go @@ -1,6 +1,7 @@ package bundle import ( + "bytes" "crypto/sha256" "fmt" @@ -185,32 +186,65 @@ func appendFile(f *File) []byte { } // encodeUnchecked writes the canonical bytes without validating b. Callers -// validate first; the decoder uses it only for the canonical re-encode check. +// validate first. func encodeUnchecked(b *Bundle) []byte { var out []byte - out = appendVarint(out, fVersion, SchemaVersion) - out = appendVarint(out, fScheme, uint64(b.Scheme)) - out = appendBytes(out, fReleaseID, b.ReleaseID[:]) - out = appendString(out, fTitle, b.Title) - out = appendVarint(out, fPosted, uint64(b.PostedUnix)) - out = appendVarint(out, fTotal, uint64(b.TotalSize)) + encodeTo(b, func(chunk []byte) bool { out = append(out, chunk...); return true }) + return out +} + +// matchesCanonical reports whether b encodes to exactly raw, comparing chunk +// by chunk so no second full-size buffer exists. +func matchesCanonical(b *Bundle, raw []byte) bool { + pos := 0 + ok := encodeTo(b, func(chunk []byte) bool { + if len(raw)-pos < len(chunk) || !bytes.Equal(raw[pos:pos+len(chunk)], chunk) { + return false + } + pos += len(chunk) + return true + }) + return ok && pos == len(raw) +} + +// encodeTo feeds the canonical encoding of b to emit in order. It stops and +// returns false as soon as emit does. The chunk is only valid during the call. +func encodeTo(b *Bundle, emit func([]byte) bool) bool { + var buf []byte + put := func(chunk []byte) bool { return emit(chunk) } + buf = appendVarint(buf[:0], fVersion, SchemaVersion) + buf = appendVarint(buf, fScheme, uint64(b.Scheme)) + buf = appendBytes(buf, fReleaseID, b.ReleaseID[:]) + buf = appendString(buf, fTitle, b.Title) + buf = appendVarint(buf, fPosted, uint64(b.PostedUnix)) + buf = appendVarint(buf, fTotal, uint64(b.TotalSize)) if b.NZBPassword != "" { - out = appendString(out, fPassword, b.NZBPassword) + buf = appendString(buf, fPassword, b.NZBPassword) + } + if !put(buf) { + return false } for _, id := range b.Pool { - out = appendString(out, fPool, id) + buf = appendString(buf[:0], fPool, id) + if !put(buf) { + return false + } } if b.Scheme == SchemeNZB { var rb []byte for _, n := range b.ReleaseFiles { rb = protowire.AppendVarint(rb, uint64(n)) } - out = appendBytes(out, fReleaseFiles, rb) + if !put(appendBytes(buf[:0], fReleaseFiles, rb)) { + return false + } } for i := range b.Files { - out = appendBytes(out, fFiles, appendFile(&b.Files[i])) + if !put(appendBytes(buf[:0], fFiles, appendFile(&b.Files[i]))) { + return false + } } - return out + return true } // Encode validates b and returns its canonical uncompressed bytes. diff --git a/internal/peernet/bundle/wire_decode.go b/internal/peernet/bundle/wire_decode.go index 4560d5112..5d1b70e3d 100644 --- a/internal/peernet/bundle/wire_decode.go +++ b/internal/peernet/bundle/wire_decode.go @@ -1,7 +1,6 @@ package bundle import ( - "bytes" "fmt" "math" @@ -612,7 +611,7 @@ func decodeRaw(raw []byte) (*Bundle, error) { // Validate each file as soon as it is decoded so a // hostile bundle fails before the next one is expanded. if val == nil { - val = &validator{b: b, names: map[string]struct{}{}} + val = &validator{b: b, names: map[string]struct{}{}, tables: map[TableKey]int32{}} err = val.checkPool() } if err == nil { @@ -649,7 +648,7 @@ func Decode(raw []byte) (*Bundle, error) { if err := Validate(b); err != nil { return nil, err } - if !bytes.Equal(encodeUnchecked(b), raw) { + if !matchesCanonical(b, raw) { return nil, fmt.Errorf("%w: not the canonical encoding", ErrMalformed) } return b, nil diff --git a/internal/peernet/bundleimport/accept.go b/internal/peernet/bundleimport/accept.go index 24a8c4623..5572ed393 100644 --- a/internal/peernet/bundleimport/accept.go +++ b/internal/peernet/bundleimport/accept.go @@ -104,6 +104,11 @@ func (i *Importer) Accept(ctx context.Context, req Request) (*Result, error) { return i.rejected(r, err) } b, loc = subset(b, loc, keep) + if req.Origin.Open { + if err := plainLayoutOnly(b); err != nil { + return i.rejected(ReasonLayoutNotAllowedOpen, err) + } + } paths := make([]string, len(loc.Files)) for n, lf := range loc.Files { @@ -133,16 +138,22 @@ func (i *Importer) Accept(ctx context.Context, req Request) (*Result, error) { } return i.rejected(ReasonProbeInconclusive, err) } - if r, err := i.verifyYenc(ctx, b); err != nil { - if r == ReasonYencMismatch && i.cfg.Reporter != nil { + if r, attributable, err := i.verifyYenc(ctx, b, req.ExpectedNZBNames); err != nil { + if attributable && i.cfg.Reporter != nil { // The layout contradicts the articles themselves: the publisher // served a bad bundle, which is attributable (fetch domain only). i.cfg.Reporter.ReportBadBundle(ctx, req.Origin, b.ReleaseID.String()) i.cnt.add(func(c *counters) { c.reports++ }) } + if attributable { + i.noteFailedBundle(ctx, req.Origin.BundleID, b.ReleaseID.String()) + } return i.rejected(r, err) } - if r, err := i.probeMedia(ctx, loc); err != nil { + if r, err := i.probeMedia(ctx, loc, req.Origin.Open); err != nil { + if r == ReasonMediaInvalid || r == ReasonStructureInvalid { + i.noteFailedBundle(ctx, req.Origin.BundleID, b.ReleaseID.String()) + } return i.rejected(r, err) } @@ -158,13 +169,48 @@ func (i *Importer) Accept(ctx context.Context, req Request) (*Result, error) { i.cnt.add(func(c *counters) { c.already++ }) return res, nil } - if err := i.write(ctx, b, loc, req, res); err != nil { + i.writeMu.RLock() + err = i.write(ctx, b, loc, req, res) + i.writeMu.RUnlock() + if err != nil { return i.rejected(ReasonWriteFailed, err) } i.cnt.add(func(c *counters) { c.accepted++ }) return res, nil } +// plainLayoutOnly requires every written file to be exactly one inline segment +// table: no outer volumes, extents, AES key or clips. Archive layouts cannot be +// bound to the articles without reading archive headers, so the open network +// does not carry them (private networks do). +func plainLayoutOnly(b *bundle.Bundle) error { + for k := range b.Files { + f := &b.Files[k] + switch { + case len(f.Outer) > 0 || len(f.Extents) > 0: + return fmt.Errorf("%s: read through outer archive volumes", f.Name) + case len(f.AESKey) > 0: + return fmt.Errorf("%s: encrypted", f.Name) + case len(f.Clips) > 0: + return fmt.Errorf("%s: clipped", f.Name) + case len(f.Segments) == 0: + return fmt.Errorf("%s: no inline segments", f.Name) + } + } + return nil +} + +// noteFailedBundle puts a layout that contradicted the articles or the media on +// the negative cache (when the reporter keeps one). +func (i *Importer) noteFailedBundle(ctx context.Context, bundleID, releaseID string) { + if bundleID == "" { + return + } + if r, ok := i.cfg.Reporter.(BundleFailedReporter); ok { + r.ReportFailedBundle(ctx, bundleID, releaseID) + } +} + // verifyIdentity recomputes every File ID and the Release ID from the layout // the bundle describes, so this package does not depend on its caller having // run bundle.Verify. @@ -278,7 +324,7 @@ func probeFiles(loc *bundle.Local) []ProbeFile { // probeMedia reads the head of the main media file(s) through the real reader // stack, in a hidden staging area that is removed on every outcome. -func (i *Importer) probeMedia(ctx context.Context, loc *bundle.Local) (Reason, error) { +func (i *Importer) probeMedia(ctx context.Context, loc *bundle.Local, open bool) (Reason, error) { var media []bundle.LocalFile for _, lf := range loc.Files { if fileinfo.IsVerifiableMediaFile(lf.Name) { @@ -310,7 +356,7 @@ func (i *Importer) probeMedia(ctx context.Context, loc *bundle.Local) (Reason, e default: return ReasonMediaInconclusive, fmt.Errorf("%s: %w", lf.Name, res.Err) } - if err := verifyStructure(ctx, st.Opener(), lf.Name, lf.Meta.FileSize); err != nil { + if err := verifyStructure(ctx, st.Opener(), lf.Name, lf.Meta.FileSize, open); err != nil { return structureReason(err), fmt.Errorf("%s: %w", lf.Name, err) } } @@ -358,6 +404,11 @@ func (i *Importer) write(ctx context.Context, b *bundle.Bundle, loc *bundle.Loca if err := i.cfg.State.Upsert(ctx, rows); err != nil { return err } + if req.Origin.BundleID != "" { + if err := i.cfg.State.SetBundleID(ctx, res.Paths, req.Origin.BundleID); err != nil { + return err + } + } if i.crashAt("rows") { return errCrashed } diff --git a/internal/peernet/bundleimport/accept_test.go b/internal/peernet/bundleimport/accept_test.go index 311e8fac6..019dd56a6 100644 --- a/internal/peernet/bundleimport/accept_test.go +++ b/internal/peernet/bundleimport/accept_test.go @@ -49,7 +49,11 @@ func (a *fakeAnnouncer) AnnounceVerifiedImport(_ context.Context, an Announcemen return nil } -type fakeReporter struct{ got, verified []string } +type fakeReporter struct{ got, verified, failedBundles []string } + +func (r *fakeReporter) ReportFailedBundle(_ context.Context, bundleID, release string) { + r.failedBundles = append(r.failedBundles, bundleID+"/"+release) +} func (r *fakeReporter) ReportVerifiedBundle(_ context.Context, o Origin, release string) { r.verified = append(r.verified, o.PublisherID+"/"+release) @@ -128,9 +132,7 @@ func mkvPlain(size int) []byte { out = append(out, append(append([]byte{}, info...), 0x84, 0, 0, 0, 0)...) fill := size - len(out) - 6 - 9 out = append(out, 0x1F, 0x43, 0xB6, 0x75, byte(0x40|fill>>8), byte(fill)) - for k := 0; k < fill; k++ { - out = append(out, byte((k*7+13)%251)) - } + out = append(out, simpleBlocks(fill)...) // a real Cluster body: SimpleBlocks out = append(out, append(append([]byte{}, cues...), 0x84, 0, 0, 0, 0)...) return out } @@ -554,3 +556,29 @@ func TestAccept_ResultCarriesFileSizesInPathOrder(t *testing.T) { assert.Equal(t, meta.FileSize, res.Sizes[n], p) } } + +// Open network: only plain layouts are accepted (no outer sources, extents, +// AES key or clips); a private network keeps accepting them. +func TestAccept_OpenNetworkRejectsEncryptedLayout(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("openaes", fileSpec{name: "movie.mkv", plain: mkvPlain(512), segSize: 128, key: testKey}) + req := e.req(b) + req.Origin.Open = true + _, err := e.imp.Accept(context.Background(), req) + require.Error(t, err) + assert.Equal(t, ReasonLayoutNotAllowedOpen, ReasonOf(err), "%v", err) + e.assertNothingBehind() + + req.Origin.Open = false + _, err = e.imp.Accept(context.Background(), req) + require.NoError(t, err, "a private network may still carry encrypted layouts") +} + +func TestAccept_OpenNetworkAcceptsPlainLayout(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("openplain", mkvSpec("movie.mkv")) + req := e.req(b) + req.Origin.Open = true + _, err := e.imp.Accept(context.Background(), req) + require.NoError(t, err) +} diff --git a/internal/peernet/bundleimport/foreignfile_test.go b/internal/peernet/bundleimport/foreignfile_test.go new file mode 100644 index 000000000..c4fcb483f --- /dev/null +++ b/internal/peernet/bundleimport/foreignfile_test.go @@ -0,0 +1,110 @@ +package bundleimport + +import ( + "context" + "testing" + "time" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// replaceWithOtherFile overwrites the metadata at path with a different file +// (what a later normal import of another layout leaves there). +func replaceWithOtherFile(t *testing.T, e *env, path string) { + t.Helper() + m := mustMeta(t, e, path) + m.SegmentData[0].Id = "someone-else@test" + require.NoError(t, e.lib.WriteFileMetadata(path, m)) +} + +// A row whose File ID is not the file at its path describes a file that is +// gone: it must never cause a delete, a hold or a forced check. +func TestForeignFile_FailedVerdictNeverDeletesTheFileAtThePath(t *testing.T) { + e := newEnv(t) + ann := e.withFailAnn() + _, res := e.accept("fo1", mkvSpec("a.mkv")) + p := res.Paths[0] + replaceWithOtherFile(t, e, p) + + e.imp.FullCheckFailed(context.Background(), p) + + assert.True(t, e.lib.FileExists(p), "the other file at the path must survive") + assert.Empty(t, ann.removed) + assert.Empty(t, ann.failed, "the queue item is not failed for a file that is not ours") + assert.Zero(t, e.rowCount(p), "the stale row is dropped") + assert.Empty(t, e.rep.got) +} + +func TestForeignFile_StaleEscalationNeverDeletesTheFileAtThePath(t *testing.T) { + e := newEnv(t) + ann := e.withFailAnn() + _, res := e.accept("fo2", mkvSpec("a.mkv")) + p := res.Paths[0] + replaceWithOtherFile(t, e, p) + e.imp.cfg.Now = func() time.Time { return time.Now().Add(MaxPendingAge + time.Hour) } + + assert.Zero(t, e.imp.EscalateStale(context.Background())) + + assert.True(t, e.lib.FileExists(p)) + assert.Empty(t, ann.removed) + assert.Zero(t, e.rowCount(p)) +} + +func TestForeignFile_IsNeitherPendingNorHeld(t *testing.T) { + e := newEnv(t) + e.withFailAnn() + _, res := e.accept("fo3", mkvSpec("a.mkv")) + p := res.Paths[0] + ctx := context.Background() + replaceWithOtherFile(t, e, p) + + assert.False(t, e.imp.IsPendingPeerImport(ctx, p), "no repair attempts are suppressed for it") + assert.NotContains(t, e.imp.PendingFullCheck(ctx, []string{p}), p, "no forced full check") + assert.Zero(t, e.rowCount(p)) +} + +func TestForeignFile_FailImportSparesForeignSiblingFiles(t *testing.T) { + e := newEnv(t) + ann := e.withFailAnn() + _, res := e.accept("fo4", mkvSpec("a.mkv"), mkvSpec("b.mkv")) + replaceWithOtherFile(t, e, res.Paths[0]) // a sibling was replaced by another file + + e.imp.FullCheckFailed(context.Background(), res.Paths[1]) + + assert.True(t, e.lib.FileExists(res.Paths[0]), "a replaced sibling is not ours to remove") + assert.False(t, e.lib.FileExists(res.Paths[1])) + assert.Equal(t, []string{res.Paths[1]}, ann.removed) + assert.Zero(t, e.rowCount(res.Paths...)) +} + +// A delete followed by a rewrite of the same File ID at the same path is a +// normal import replacing the pending peer import, not a rename: the peer +// import ends cleanly and the hold is released. +func TestPathObserver_NormalImportOverAPendingImportEndsIt(t *testing.T) { + e := newEnv(t) + ann := e.withFailAnn() + _, res := e.accept("fo5", mkvSpec("a.mkv")) + p := res.Paths[0] + ctx := context.Background() + repo := database.NewRepository(e.db.Connection(), database.DialectSQLite) + item := &database.ImportQueueItem{NzbPath: "x.nzb", Status: database.QueueStatusCompleted, Priority: database.QueuePriorityNormal} + require.NoError(t, repo.AddToQueue(ctx, item)) + _, err := e.db.Connection().Exec(`UPDATE import_queue SET id = 11 WHERE id = ?`, item.ID) // the import is bound to queue item 11 + require.NoError(t, err) + require.NoError(t, e.repo.SetQueueHold(ctx, 11, true)) + obs := e.imp.PathObserver() + m := mustMeta(t, e, p) + require.NoError(t, e.lib.DeleteFileMetadata(ctx, p)) + obs.FileDeleted(ctx, p) + require.NoError(t, e.lib.WriteFileMetadata(p, m)) // the normal import writes the same content + obs.FileWritten(ctx, p) + obs.Flush() + + assert.Zero(t, e.rowCount(p), "the row must not attach to a normal import") + assert.NotContains(t, e.imp.PendingFullCheck(ctx, []string{p}), p, "no forced check, no hold") + assert.Equal(t, []int64{11}, ann.freed, "the hold is released") + assert.True(t, e.lib.FileExists(p), "the normal import's file stays") + assert.Empty(t, e.ann.got) +} diff --git a/internal/peernet/bundleimport/headers_test.go b/internal/peernet/bundleimport/headers_test.go new file mode 100644 index 000000000..2a7bce619 --- /dev/null +++ b/internal/peernet/bundleimport/headers_test.go @@ -0,0 +1,129 @@ +package bundleimport + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestVerifyPeerHeaders_AgreeingSamplePasses(t *testing.T) { + e := newEnv(t) + _, res := e.accept("vh1", mkvSpec("a.mkv")) + require.NoError(t, e.imp.VerifyPeerHeaders(context.Background(), res.Paths[0])) +} + +func TestVerifyPeerHeaders_LayoutLieFailsAndIsAttributable(t *testing.T) { + e := newEnv(t) + b, res := e.accept("vh2", mkvSpec("a.mkv")) + // The lie sits in an interior article the accept-time sample may have + // missed in a bigger release; here the whole file is sampled. + beh := e.fp.BehaviorFor(b.Pool[2]) + beh.YEnc.PartBegin += 9 + e.fp.SetBehavior(b.Pool[2], beh) + ctx := context.Background() + + err := e.imp.VerifyPeerHeaders(ctx, res.Paths[0]) + var lm *LayoutMismatchError + require.ErrorAs(t, err, &lm) + + e.imp.FullCheckFailed(ctx, res.Paths[0]) + assert.Equal(t, []string{strings.Repeat("d", 64) + "/" + b.ReleaseID.String()}, e.rep.got, "a header contradiction is the publisher's") +} + +func TestVerifyPeerHeaders_NotAPendingImportIsIgnored(t *testing.T) { + e := newEnv(t) + require.NoError(t, e.imp.VerifyPeerHeaders(context.Background(), "movies/none/x.mkv")) +} + +func TestHeaderSampleSize(t *testing.T) { + assert.Equal(t, 5, headerSampleSize(5)) + assert.Equal(t, 20, headerSampleSize(1000)) + assert.Equal(t, 100, headerSampleSize(10000)) + assert.Equal(t, 500, headerSampleSize(400000)) +} + +// A degraded verdict fails the import at once; with the user's NZB at hand +// the queue item is put back for a normal import, after the files are gone. +type requeueAnn struct { + *failAnn + requeued []int64 + fileGoneAtReq bool + can bool + path string + e *env +} + +func (a *requeueAnn) RequeueForNormalImport(_ context.Context, id int64) (bool, error) { + a.requeued = append(a.requeued, id) + a.fileGoneAtReq = !a.e.lib.FileExists(a.path) + return a.can, nil +} + +func TestDegraded_FastPathItemIsRequeuedForANormalImportAfterRemoval(t *testing.T) { + e := newEnv(t) + base := e.withFailAnn() + _, res := e.accept("dg1", mkvSpec("a.mkv")) + ann := &requeueAnn{failAnn: base, can: true, path: res.Paths[0], e: e} + e.imp.cfg.Announcer = ann + ctx := context.Background() + + e.imp.FullCheckDegraded(ctx, res.Paths[0]) + + assert.Equal(t, []int64{11}, ann.requeued) + assert.True(t, ann.fileGoneAtReq, "the normal import must not race the removal") + assert.Empty(t, ann.failed, "the item is not marked failed: the user still gets their release") + assert.Zero(t, e.rowCount(res.Paths...)) + assert.Empty(t, e.rep.got, "missing articles are not the publisher's fault") + assert.Empty(t, ann.got, "never announced") +} + +func TestDegraded_DirectImportIsFailedNotRequeued(t *testing.T) { + e := newEnv(t) + base := e.withFailAnn() + _, res := e.accept("dg2", mkvSpec("a.mkv")) + ann := &requeueAnn{failAnn: base, can: false, path: res.Paths[0], e: e} + e.imp.cfg.Announcer = ann + + e.imp.FullCheckDegraded(context.Background(), res.Paths[0]) + + assert.Equal(t, failReasonDegraded, ann.failed[11]) + assert.False(t, e.lib.FileExists(res.Paths[0])) +} + +// A layout that failed local verification goes on the negative cache under its +// Bundle ID, so the same layout from another publisher is skipped. +func TestNegativeCache_AcceptTimeContradictionAndFullCheckMismatchAreReported(t *testing.T) { + bid := strings.Repeat("e", 64) + + e := newEnv(t) + b := e.mkBundle("nc1", mkvSpec("movie.mkv")) + beh := e.fp.BehaviorFor(b.Pool[1]) + beh.YEnc.PartSize++ + e.fp.SetBehavior(b.Pool[1], beh) + req := e.req(b) + req.Origin.BundleID = bid + _, err := e.imp.Accept(context.Background(), req) + require.Error(t, err) + assert.Equal(t, []string{bid + "/" + b.ReleaseID.String()}, e.rep.failedBundles) + + e2 := newEnv(t) + b2, res := e2.accept("nc2", mkvSpec("a.mkv")) + _ = b2 + require.NoError(t, e2.repo.SetBundleID(context.Background(), res.Paths, bid)) + beh = e2.fp.BehaviorFor(b2.Pool[2]) + beh.YEnc.PartBegin += 3 + e2.fp.SetBehavior(b2.Pool[2], beh) + require.Error(t, e2.imp.VerifyPeerHeaders(context.Background(), res.Paths[0])) + e2.imp.FullCheckFailed(context.Background(), res.Paths[0]) + assert.Equal(t, []string{bid + "/" + b2.ReleaseID.String()}, e2.rep.failedBundles) + + // A missing-only failure does not blacklist the layout. + e3 := newEnv(t) + _, res3 := e3.accept("nc3", mkvSpec("a.mkv")) + require.NoError(t, e3.repo.SetBundleID(context.Background(), res3.Paths, bid)) + e3.imp.FullCheckFailed(context.Background(), res3.Paths[0]) + assert.Empty(t, e3.rep.failedBundles) +} diff --git a/internal/peernet/bundleimport/holds_test.go b/internal/peernet/bundleimport/holds_test.go new file mode 100644 index 000000000..66357de99 --- /dev/null +++ b/internal/peernet/bundleimport/holds_test.go @@ -0,0 +1,99 @@ +package bundleimport + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/kipsilabs/altmount/internal/database" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" +) + +// A hold whose rows are all passed and committed, and for which no ARR is +// owed, is released by housekeeping even when the release at pass time failed. +func TestHolds_SettledHoldIsReleasedByHousekeeping(t *testing.T) { + e := newEnv(t) + ann := e.withFailAnn() + ctx := context.Background() + b := e.mkBundle("hs", mkvSpec("a.mkv")) + req := e.req(b) + req.Target.SkipArr = true + res, err := e.imp.Accept(ctx, req) + require.NoError(t, err) + q := database.NewRepository(e.db.Connection(), database.DialectSQLite) + item := &database.ImportQueueItem{NzbPath: "x.nzb", Status: database.QueueStatusCompleted, Priority: database.QueuePriorityNormal} + require.NoError(t, q.AddToQueue(ctx, item)) + _, err = e.db.Connection().Exec(`UPDATE import_queue SET id = 11 WHERE id = ?`, item.ID) + require.NoError(t, err) + require.NoError(t, e.repo.SetQueueHold(ctx, 11, true)) + _, err = e.repo.MarkPassed(ctx, res.Paths[0]) // passed, but the hold release never happened + require.NoError(t, err) + + e.imp.releaseOrphanHolds(ctx) + + assert.Equal(t, []int64{11}, ann.freed) +} + +func TestHolds_PendingHoldIsKept(t *testing.T) { + e := newEnv(t) + ann := e.withFailAnn() + ctx := context.Background() + b := e.mkBundle("hp", mkvSpec("a.mkv")) + req := e.req(b) + req.Target.SkipArr = true + _, err := e.imp.Accept(ctx, req) + require.NoError(t, err) + q := database.NewRepository(e.db.Connection(), database.DialectSQLite) + item := &database.ImportQueueItem{NzbPath: "x.nzb", Status: database.QueueStatusCompleted, Priority: database.QueuePriorityNormal} + require.NoError(t, q.AddToQueue(ctx, item)) + _, err = e.db.Connection().Exec(`UPDATE import_queue SET id = 11 WHERE id = ?`, item.ID) + require.NoError(t, err) + require.NoError(t, e.repo.SetQueueHold(ctx, 11, true)) + + e.imp.releaseOrphanHolds(ctx) + + assert.Empty(t, ann.freed, "a pending import keeps its hold") +} + +type blockingLibrary struct { + Library + entered chan struct{} + release chan struct{} + once sync.Once +} + +func (l *blockingLibrary) WriteFileMetadata(p string, m *metapb.FileMetadata) error { + l.once.Do(func() { close(l.entered); <-l.release }) + return l.Library.WriteFileMetadata(p, m) +} + +// Reconcile rolls back every uncommitted row, so it must wait for an Accept +// that is writing and never roll its rows back. +func TestReconcile_NeverRollsBackAnInFlightAccept(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("rc", mkvSpec("a.mkv")) + lib := &blockingLibrary{Library: e.lib, entered: make(chan struct{}), release: make(chan struct{})} + e.imp.cfg.Library = lib + done := make(chan error, 1) + go func() { _, err := e.imp.Accept(context.Background(), e.req(b)); done <- err }() + <-lib.entered // rows are written (uncommitted), the first file is about to be + + reconciled := make(chan struct{}) + go func() { e.imp.Reconcile(context.Background()); close(reconciled) }() + select { + case <-reconciled: + t.Fatal("Reconcile ran while an Accept was writing") + case <-time.After(300 * time.Millisecond): + } + close(lib.release) + require.NoError(t, <-done) + <-reconciled + row, ok, _ := e.repo.Get(context.Background(), "movies/Test.Release/a.mkv") + require.True(t, ok, "the finished import keeps its row") + assert.True(t, row.Committed) + assert.True(t, e.lib.FileExists("movies/Test.Release/a.mkv")) +} diff --git a/internal/peernet/bundleimport/importer.go b/internal/peernet/bundleimport/importer.go index e962a95de..670f10853 100644 --- a/internal/peernet/bundleimport/importer.go +++ b/internal/peernet/bundleimport/importer.go @@ -35,6 +35,32 @@ type Config struct { Now func() time.Time } +// Missing lists the collaborators the accept pipeline cannot work without and +// that are nil. A production wiring must call it and refuse to run with any +// missing (a nil Headers, for example, rejects every bundle at the first +// sample): optional pieces are only Config.AllowedExtensions and +// Config.WillBeChecked (nil means "no restriction" and "every path"). +func (c Config) Missing() []string { + var out []string + add := func(nilDep bool, name string) { + if nilDep { + out = append(out, name) + } + } + add(c.Library == nil, "Library") + add(c.Stager == nil, "Stager") + add(c.Prober == nil, "Prober") + add(c.State == nil, "State") + add(c.Index == nil, "Index") + add(c.Fetcher == nil, "Fetcher") + add(c.Headers == nil, "Headers") + add(c.Recorder == nil, "Recorder") + add(c.Announcer == nil, "Announcer") + add(c.Reporter == nil, "Reporter") + add(c.HealthEnabled == nil, "HealthEnabled") + return out +} + // Importer accepts bundles. It is safe for concurrent use. type Importer struct { cfg Config @@ -43,6 +69,12 @@ type Importer struct { // reportedMu guards reported. reportedMu sync.Mutex flight singleflight.Group + // mismatched holds the paths whose header sample contradicted the layout. + mismatched sync.Map + + // writeMu: a write holds it shared from its first state row to its commit; + // Reconcile takes it exclusively, so it can never roll back a live write. + writeMu sync.RWMutex tmu sync.Mutex targets map[string]*targetLock diff --git a/internal/peernet/bundleimport/pathobserver.go b/internal/peernet/bundleimport/pathobserver.go index c0bac0f8c..39b448723 100644 --- a/internal/peernet/bundleimport/pathobserver.go +++ b/internal/peernet/bundleimport/pathobserver.go @@ -58,7 +58,16 @@ func (o *PathObserver) FileWritten(ctx context.Context, path string) { delete(o.stash, id.String()) } o.mu.Unlock() - if !ok || s.path == path { + if !ok { + return + } + if s.path == path { + // Deleted and written again at the same path: something other than a + // rename wrote it (a normal import of the same content, or the user). + // It must not inherit the pending state: the peer import ends cleanly, + // its hold is released and the file is an ordinary one. + _ = o.i.cfg.State.Delete(ctx, []string{path}) + o.i.releaseOrphanHolds(ctx) return } if _, err := o.i.cfg.State.MoveFile(ctx, s.path, path); err != nil { diff --git a/internal/peernet/bundleimport/reconcile.go b/internal/peernet/bundleimport/reconcile.go index 8aebcc37c..b5101472c 100644 --- a/internal/peernet/bundleimport/reconcile.go +++ b/internal/peernet/bundleimport/reconcile.go @@ -37,6 +37,9 @@ func (i *Importer) Reconcile(ctx context.Context) { } func (i *Importer) rollbackUncommitted(ctx context.Context) { + // Every uncommitted row is rolled back, so no write may be in flight. + i.writeMu.Lock() + defer i.writeMu.Unlock() rows, err := i.cfg.State.ListUncommitted(ctx, 5000) if err != nil { slog.WarnContext(ctx, "could not list unfinished peer imports", "error", err) diff --git a/internal/peernet/bundleimport/sample.go b/internal/peernet/bundleimport/sample.go index 1dacefa5a..1684fd6b2 100644 --- a/internal/peernet/bundleimport/sample.go +++ b/internal/peernet/bundleimport/sample.go @@ -7,10 +7,12 @@ import ( "errors" "fmt" "io" + "strings" "sync" "github.com/javi11/nntppool/v5" + "github.com/kipsilabs/altmount/internal/importer/parser/fileinfo" "github.com/kipsilabs/altmount/internal/peernet/bundle" "github.com/kipsilabs/altmount/internal/pool" ) @@ -19,6 +21,8 @@ import ( // the decoded size of the part, its 0-based offset in the file and the size of // the whole file. type YencHeader struct { + // Name is the file name of the =ybegin header ("" when absent). + Name string PartBegin int64 PartSize int64 FileSize int64 @@ -34,7 +38,7 @@ type poolHeaders struct{ p pool.Manager } // NewPoolHeaderReader reads headers through the connection pool. The pool // cannot stop after the headers, so a read costs the whole article (about -// 0.75 MB); the sample count is therefore bounded by Policy.YencSamples. +// 0.75 MB); the sample count and bytes are therefore bounded by Policy. func NewPoolHeaderReader(p pool.Manager) SegmentHeaders { return poolHeaders{p: p} } func (h poolHeaders) SegmentHeader(ctx context.Context, id string) (YencHeader, error) { @@ -46,7 +50,7 @@ func (h poolHeaders) SegmentHeader(ctx context.Context, id string) (YencHeader, if err != nil { return YencHeader{}, err } - return YencHeader{PartBegin: body.YEnc.PartBegin, PartSize: body.YEnc.PartSize, FileSize: body.YEnc.FileSize}, nil + return YencHeader{Name: body.YEnc.FileName, PartBegin: body.YEnc.PartBegin, PartSize: body.YEnc.PartSize, FileSize: body.YEnc.FileSize}, nil } // sampleTable is one segment table read as a whole file or archive volume. @@ -56,25 +60,50 @@ type sampleTable struct { sizes []int64 prefix []int64 // prefix[i] = sum of sizes[:i] total int64 + key bundle.TableKey + // main marks the table of a media file (denser sampling). + main bool + // nzbName is the file name the user's NZB subject gives the NZB entry this + // table reads ("" when unknown); the first article's yEnc name must agree. + nzbName string } type sample struct{ table, pos int } -// tablesOf lists the file and outer-volume tables of a bundle, each outer -// volume once even when several files share it. -func tablesOf(b *bundle.Bundle) []sampleTable { +// entryNames maps each pool position of a nzb-scheme bundle to the subject +// file name of its NZB entry ("" when there is none). +func entryNameAt(b *bundle.Bundle, names []string, pos int64) string { + if len(names) != len(b.ReleaseFiles) { + return "" + } + var at int64 + for e, n := range b.ReleaseFiles { + if pos < at+int64(n) { + return names[e] + } + at += int64(n) + } + return "" +} + +// tablesOf lists the file and outer-volume tables of a bundle. A table is +// identified by the hash of its whole reference list (with sizes and ranges), +// so a table shared by several files is sampled once and two different tables +// never collapse because they start with the same article. +func tablesOf(b *bundle.Bundle, names []string) []sampleTable { var out []sampleTable - seen := map[string]struct{}{} - add := func(name string, segs []bundle.Segment) { + seen := map[bundle.TableKey]struct{}{} + add := func(name string, segs []bundle.Segment, main bool) { if len(segs) == 0 { return } - first := b.Pool[segs[0].Ref] - if _, dup := seen[first]; dup { + key := bundle.KeyOf(segs) + if _, dup := seen[key]; dup { return } - seen[first] = struct{}{} - t := sampleTable{name: name, ids: make([]string, len(segs)), sizes: make([]int64, len(segs)), prefix: make([]int64, len(segs))} + seen[key] = struct{}{} + t := sampleTable{name: name, key: key, main: main, nzbName: entryNameAt(b, names, segs[0].Ref), + ids: make([]string, len(segs)), sizes: make([]int64, len(segs)), prefix: make([]int64, len(segs))} for i, s := range segs { t.ids[i], t.sizes[i], t.prefix[i] = b.Pool[s.Ref], s.Size, t.total t.total += s.Size @@ -83,9 +112,9 @@ func tablesOf(b *bundle.Bundle) []sampleTable { } for i := range b.Files { f := &b.Files[i] - add(f.Name, f.Segments) + add(f.Name, f.Segments, fileinfo.IsVerifiableMediaFile(f.Name)) for j := range f.Outer { - add(fmt.Sprintf("%s (volume %d)", f.Name, j+1), f.Outer[j].Segments) + add(fmt.Sprintf("%s (volume %d)", f.Name, j+1), f.Outer[j].Segments, false) } } return out @@ -117,60 +146,137 @@ func shuffle[T any](s []T) { } } -// planSamples picks, per table, the first and last segment and perRandom -// random ones in between, within a total cap. When the cap is smaller than the -// number of tables, a random subset of tables is read. -func planSamples(tables []sampleTable, perRandom, limit int) []sample { - order := make([]int, len(tables)) - for i := range order { - order[i] = i +// errTooManyTables means the minimum sampling rate does not fit the budget. +var errTooManyTables = errors.New("more segment tables than the sampling budget covers") + +// interiorFor returns how many random interior articles a table gets at least +// (one) and in total: tables of media files get one per 500 segments, at least +// three. +func interiorFor(t sampleTable) (minimum, total int) { + n := len(t.ids) - 2 + if n <= 0 { + return 0, 0 } - shuffle(order) + total = 1 + if t.main { + total = max(3, len(t.ids)/500) + } + return 1, min(total, n) +} + +// planSamples picks, per table, the first and last segment and at least one +// uniformly random interior one (more for the tables of media files), within a +// cap on the number of articles and on the bytes read. A budget that cannot +// cover every table at the minimum rate is an error, never a silent skip. +func planSamples(tables []sampleTable, limit int, maxBytes int64) ([]sample, error) { var plan []sample + var bytes int64 taken := map[sample]struct{}{} - push := func(t, pos int) { + push := func(t, pos int) bool { s := sample{t, pos} - if _, dup := taken[s]; dup || len(plan) >= limit { - return + if _, dup := taken[s]; dup { + return false } taken[s] = struct{}{} plan = append(plan, s) + bytes += tables[t].sizes[pos] + return true } - for _, t := range order { + pick := func(t, count int) { + n := len(tables[t].ids) - 2 + for k, tries := 0, 0; k < count && tries < 32*count+32; tries++ { + if push(t, 1+randN(n)) { + k++ + } + } + } + extra := make([]int, len(tables)) + for t := range tables { n := len(tables[t].ids) push(t, 0) if n > 1 { push(t, n-1) } + lo, hi := interiorFor(tables[t]) + pick(t, lo) + extra[t] = hi - lo + } + if len(plan) > limit || bytes > maxBytes { + return nil, fmt.Errorf("%w: %d tables need %d articles (%d bytes), budget %d articles and %d bytes", + errTooManyTables, len(tables), len(plan), bytes, limit, maxBytes) + } + order := make([]int, len(tables)) + for i := range order { + order[i] = i } + shuffle(order) for _, t := range order { - n := len(tables[t].ids) - for k, tries := 0, 0; k < perRandom && n > 2 && tries < 16*perRandom; tries++ { - before := len(plan) - push(t, 1+randN(n-2)) - if len(plan) > before { - k++ - } + for k := 0; k < extra[t] && len(plan) < limit && bytes+tables[t].sizes[0] <= maxBytes; k++ { + pick(t, 1) } } - return plan + return plan, nil +} + +// SubjectFilename returns the quoted file name of an NZB subject, or "". +func SubjectFilename(subject string) string { + j := strings.LastIndexByte(subject, '"') + if j <= 0 { + return "" + } + i := strings.LastIndexByte(subject[:j], '"') + if i < 0 { + return "" + } + return strings.TrimSpace(subject[i+1 : j]) } // verifyYenc reads the yEnc headers of sampled articles and requires each to // agree with the layout: decoded part size equals the declared size, the part // begins at the cumulative offset the layout implies, and the file total equals -// the sum of the table's sizes. A mismatch is the publisher's doing. -func (i *Importer) verifyYenc(ctx context.Context, b *bundle.Bundle) (Reason, error) { +// the sum of the table's sizes. When the user's NZB is present the first +// article's yEnc file name must also agree with the subject file name of the +// NZB entry the table reads. This is sampling, not proof: the bytes between +// sampled articles are unchecked. attributable says the publisher's layout +// contradicts the articles (not merely the local NZB, or a missing article). +func (i *Importer) verifyYenc(ctx context.Context, b *bundle.Bundle, names []string) (reason Reason, attributable bool, err error) { if i.cfg.Headers == nil { - return ReasonProbeInconclusive, errors.New("no way to read article headers") + return ReasonProbeInconclusive, false, errors.New("no way to read article headers") + } + tables := tablesOf(b, names) + plan, perr := planSamples(tables, i.cfg.Policy.YencSamples(), i.cfg.Policy.YencBytes()) + if perr != nil { + return ReasonTooManyTables, false, perr + } + ctx, cancel := context.WithTimeout(ctx, i.cfg.Policy.YencTimeoutOrDefault()) + defer cancel() + results := i.runSamples(ctx, tables, plan) + var first *sampleVerdict + for k := range results { + r := &results[k] + if r.reason == ReasonYencMismatch { + return r.reason, r.attributable, r.err // a contradiction outranks a missing article + } + if r.reason != "" && first == nil { + first = r + } } - tables := tablesOf(b) - plan := planSamples(tables, i.cfg.Policy.YencRandom(), i.cfg.Policy.YencSamples()) - type verdict struct { - reason Reason - err error + if first != nil { + return first.reason, false, first.err } - results := make([]verdict, len(plan)) + return "", false, nil +} + +type sampleVerdict struct { + reason Reason + err error + attributable bool +} + +// runSamples reads the planned articles' headers (four at a time) and compares +// each with the layout of its table. +func (i *Importer) runSamples(ctx context.Context, tables []sampleTable, plan []sample) []sampleVerdict { + results := make([]sampleVerdict, len(plan)) var wg sync.WaitGroup sem := make(chan struct{}, 4) for k, s := range plan { @@ -183,28 +289,17 @@ func (i *Importer) verifyYenc(ctx context.Context, b *bundle.Bundle) (Reason, er h, err := i.cfg.Headers.SegmentHeader(ctx, t.ids[s.pos]) switch { case errors.Is(err, nntppool.ErrArticleNotFound): - results[k] = verdict{ReasonProbeMissing, fmt.Errorf("%s: article %d is missing", t.name, s.pos)} + results[k] = sampleVerdict{reason: ReasonProbeMissing, err: fmt.Errorf("%s: article %d is missing", t.name, s.pos)} case err != nil: - results[k] = verdict{ReasonProbeInconclusive, fmt.Errorf("%s: %w", t.name, err)} + results[k] = sampleVerdict{reason: ReasonProbeInconclusive, err: fmt.Errorf("%s: %w", t.name, err)} case h.PartSize != t.sizes[s.pos] || h.PartBegin != t.prefix[s.pos] || h.FileSize != t.total: - results[k] = verdict{ReasonYencMismatch, fmt.Errorf("%s: article %d says part %d at %d of %d, layout says %d at %d of %d", + results[k] = sampleVerdict{reason: ReasonYencMismatch, attributable: true, err: fmt.Errorf("%s: article %d says part %d at %d of %d, layout says %d at %d of %d", t.name, s.pos, h.PartSize, h.PartBegin, h.FileSize, t.sizes[s.pos], t.prefix[s.pos], t.total)} + case s.pos == 0 && t.nzbName != "" && h.Name != "" && !strings.EqualFold(h.Name, t.nzbName): + results[k] = sampleVerdict{reason: ReasonYencMismatch, err: fmt.Errorf("%s: first article is named %q, your NZB names this entry %q", t.name, h.Name, t.nzbName)} } }() } wg.Wait() - var first *verdict - for k := range results { - r := &results[k] - if r.reason == ReasonYencMismatch { - return r.reason, r.err // a contradiction outranks a missing article - } - if r.reason != "" && first == nil { - first = r - } - } - if first != nil { - return first.reason, first.err - } - return "", nil + return results } diff --git a/internal/peernet/bundleimport/structure.go b/internal/peernet/bundleimport/structure.go index 55db1ba08..4bda6b888 100644 --- a/internal/peernet/bundleimport/structure.go +++ b/internal/peernet/bundleimport/structure.go @@ -51,19 +51,41 @@ const ( var errStructure = errors.New("media structure is not valid") +// errNotVerifiable says the container has no structural check here: the open +// network does not carry such media. +var errNotVerifiable = errors.New("container has no structural check") + type readerAt struct { ctx context.Context f io.ReaderAt size int64 n int // reads made, bounded + hn int // box-header reads made, bounded } const maxStructReads = 64 +// maxHeaderReads bounds the 16-byte box-header reads of a top-level MP4 walk, +// counted apart from the other reads so a legitimate file with many top-level +// boxes is not rejected for the number of reads. +const maxHeaderReads = 256 + func (r *readerAt) read(off int64, n int) ([]byte, error) { if r.n++; r.n > maxStructReads { return nil, fmt.Errorf("%w: too many reads", errStructure) } + return r.readRaw(off, n) +} + +// readHeader reads a box header (at most 16 bytes) against its own budget. +func (r *readerAt) readHeader(off int64) ([]byte, error) { + if r.hn++; r.hn > maxHeaderReads { + return nil, fmt.Errorf("%w: too many top-level boxes", errStructure) + } + return r.readRaw(off, 16) +} + +func (r *readerAt) readRaw(off int64, n int) ([]byte, error) { if off < 0 || off >= r.size { return nil, fmt.Errorf("%w: offset %d outside the file", errStructure, off) } @@ -80,7 +102,7 @@ func (r *readerAt) read(off int64, n int) ([]byte, error) { } // verifyStructure runs the container check for the staged file name. -func verifyStructure(ctx context.Context, op contentverify.Opener, name string, size int64) error { +func verifyStructure(ctx context.Context, op contentverify.Opener, name string, size int64, open bool) error { ctx, cancel := context.WithTimeout(ctx, structReadTimeout) defer cancel() ctx = context.WithValue(ctx, utils.MaxPrefetchKey, 1) @@ -103,6 +125,9 @@ func verifyStructure(ctx context.Context, op contentverify.Opener, name string, case tsStride(r) != 0: return verifyTS(r, tsStride(r)) } + if open { + return fmt.Errorf("%w: %s", errNotVerifiable, name) + } return verifyWindows(r) } @@ -146,7 +171,7 @@ func verifyMP4(r *readerAt) error { if boxes > maxBoxes { return fmt.Errorf("%w: too many boxes", errStructure) } - hdr, err := r.read(off, 16) + hdr, err := r.readHeader(off) if err != nil { return err } @@ -223,9 +248,11 @@ func (r *readerAt) tailIsZero(off int64) bool { return true } -// verifyFragmentedTail requires a fragment-family box that ends exactly at the -// end of the file, found in a tail window. Random bytes at the end do not -// satisfy it: a header with a known type whose size lands on the last byte. +// verifyFragmentedTail requires the end of a fragmented file to be a real +// fragment-family structure, found by walking backwards from the last byte: +// optionally free/skip boxes, then either an mfra whose mfro states the mfra's +// own size, or an mdat directly preceded by a moof that ends exactly where the +// mdat starts. Random bytes, or one 8-byte box header, do not satisfy it. func verifyFragmentedTail(r *readerAt) error { for _, w := range fragTailWindows { n := min(w, r.size) @@ -234,20 +261,66 @@ func verifyFragmentedTail(r *readerAt) error { if err != nil { return err } - for o := 0; o+8 <= len(buf); o++ { - size := int64(uint32(buf[o])<<24 | uint32(buf[o+1])<<16 | uint32(buf[o+2])<<8 | uint32(buf[o+3])) - if size < 8 || int64(o)+size != int64(len(buf)) { - continue - } - if fragmentBoxTypes[string(buf[o+4:o+8])] { - return nil - } + if tailIsFragmentFamily(buf) { + return nil } if n == r.size { break } } - return fmt.Errorf("%w: no box ends at the end of the fragmented file", errStructure) + return fmt.Errorf("%w: the fragmented file does not end in a moof+mdat pair or an mfra", errStructure) +} + +func be32(b []byte) int { + return int(uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3])) +} + +// boxEndingAt finds the smallest fragment-family box in buf that ends exactly +// at end, scanning backwards. +func boxEndingAt(buf []byte, end int) (off int, typ string, ok bool) { + for o := end - 8; o >= 0; o-- { + size := be32(buf[o:]) + if size < 8 || o+size != end { + continue + } + if t := string(buf[o+4 : o+8]); fragmentBoxTypes[t] { + return o, t, true + } + } + return 0, "", false +} + +func tailIsFragmentFamily(buf []byte) bool { + end := len(buf) + for k := 0; k < 4; k++ { // trailing free/skip padding + o, t, ok := boxEndingAt(buf, end) + if !ok || (t != "free" && t != "skip") { + break + } + end = o + } + o, t, ok := boxEndingAt(buf, end) + if !ok { + return false + } + switch t { + case "mfra": + return mfraIsValid(buf[o:end]) + case "mdat": + _, pt, pok := boxEndingAt(buf, o) + return pok && pt == "moof" + } + return false +} + +// mfraIsValid checks that the last 16 bytes of an mfra are an mfro whose size +// field equals the mfra's own size. +func mfraIsValid(mfra []byte) bool { + if len(mfra) < 8+16 { + return false + } + mfro := mfra[len(mfra)-16:] + return be32(mfro) == 16 && string(mfro[4:8]) == "mfro" && be32(mfro[12:]) == len(mfra) } func printableBoxType(t string) bool { @@ -279,6 +352,9 @@ func tsStride(r *readerAt) int64 { return 0 } +// tsRandomChecks is how many random packet positions are checked for sync. +const tsRandomChecks = 8 + func verifyTS(r *readerAt, stride int64) error { sync := int64(0) if stride == 192 { @@ -293,6 +369,12 @@ func verifyTS(r *readerAt, stride int64) error { return err } } + // Positions the publisher cannot predict, on the packet grid. + for range tsRandomChecks { + if err := tsSync(r, int64(randN(int(min(packets, 1<<30))))*stride+sync); err != nil { + return err + } + } return tsSync(r, (packets-1)*stride+sync) } @@ -377,7 +459,10 @@ func verifyMatroska(r *readerAt) error { return fmt.Errorf("%w: Segment ends at %d, file is %d bytes", errStructure, end, r.size) } } - return verifySeekHead(r, start) + if err := verifySeekHead(r, start); err != nil { + return err + } + return verifyMatroskaInterior(r, start) } // level1IDs are the elements that can start a Segment's children. @@ -493,6 +578,8 @@ func parseSeek(b []byte) (id []byte, pos int64, err error) { // structureReason maps a structure error to the rejection reason. func structureReason(err error) Reason { switch { + case errors.Is(err, errNotVerifiable): + return ReasonContainerNotVerifiable case errors.Is(err, errStructure): return ReasonStructureInvalid case errors.Is(err, nntppool.ErrArticleNotFound): @@ -500,3 +587,87 @@ func structureReason(err error) Reason { } return ReasonMediaInconclusive } + +// ---- Matroska interior ---- + +// Interior elements that follow one another inside Clusters. +var interiorIDs = map[uint64]bool{ + 0xA3: true /* SimpleBlock */, 0xA0: true /* BlockGroup */, 0xE7: true /* Timecode */, 0x1F43B675: true, /* Cluster */ +} + +const ( + interiorWindow = 1 << 20 // bytes read at one interior position + interiorTries = 3 // fresh random positions per check before giving up + interiorChecks = 3 // interior positions that must parse + interiorWholeMax = 256 << 10 +) + +// interiorChain reports whether win holds two consecutive interior elements +// (an element header whose size lands on the header of another interior +// element, or a Cluster header followed by one). Zeros and random bytes do not +// form such a chain. +func interiorChain(win []byte) bool { + for i := 0; i+16 < len(win); i++ { + id, idn, _, ok := ebmlVint(win[i:], true) + if !ok || !interiorIDs[id] { + continue + } + sz, sn, unknown, ok := ebmlVint(win[i+idn:], false) + if !ok || unknown { + continue + } + next := int64(i + idn + sn) + if id != 0x1F43B675 { // a Cluster is entered, anything else is skipped + next += int64(sz) + } + if next < 0 || next+8 > int64(len(win)) { + continue + } + id2, _, _, ok := ebmlVint(win[next:], true) + if ok && interiorIDs[id2] { + return true + } + } + return false +} + +// verifyMatroskaInterior reads a few random interior positions and requires +// each to parse as Matroska interior structure. The SeekHead and the tail alone +// cannot tell a real file from a valid head and tail around garbage. Inside a +// frame nothing can be checked, so this proves the positions read, not the +// bytes between them. +func verifyMatroskaInterior(r *readerAt, segStart int64) error { + if r.size <= interiorWholeMax { + win, err := r.read(segStart, int(min(r.size-segStart, int64(interiorWholeMax)))) + if err != nil { + return err + } + if !interiorChain(win) { + return fmt.Errorf("%w: no Matroska cluster data in the body of the file", errStructure) + } + return nil + } + lo := r.size / 8 + hi := r.size - tailWindow - interiorWindow + if hi <= lo { + lo, hi = r.size/4, r.size/2 + } + for range interiorChecks { + found := false + for range interiorTries { + off := lo + int64(randN(int(min(hi-lo, 1<<30)))) + win, err := r.read(off, interiorWindow) + if err != nil { + return err + } + if interiorChain(win) { + found = true + break + } + } + if !found { + return fmt.Errorf("%w: a random position inside the file holds no Matroska cluster data", errStructure) + } + } + return nil +} diff --git a/internal/peernet/bundleimport/structure_forgery_test.go b/internal/peernet/bundleimport/structure_forgery_test.go new file mode 100644 index 000000000..dcafc2094 --- /dev/null +++ b/internal/peernet/bundleimport/structure_forgery_test.go @@ -0,0 +1,170 @@ +package bundleimport + +import ( + "bytes" + "context" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func structureOpen(t *testing.T, data []byte, name string, open bool) error { + t.Helper() + fs := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, name, data, 0o644)) + return verifyStructure(context.Background(), memOpener{fs}, name, int64(len(data)), open) +} + +func box(typ string, payload []byte) []byte { + n := 8 + len(payload) + return append(append([]byte{byte(n >> 24), byte(n >> 16), byte(n >> 8), byte(n)}, typ...), payload...) +} + +// Reviewer forgery: valid fragmented head, 3 MB of garbage, then any 8 bytes +// shaped like a `free` box. It used to pass on the free box alone. +func TestForgery_FragmentedMP4WithGarbageMiddleAndFreeTail(t *testing.T) { + var b []byte + b = append(b, box("ftyp", make([]byte, 8))...) + b = append(b, box("moov", make([]byte, 100))...) + b = append(b, box("mdat", make([]byte, 100))...) + for i := 0; i < 8; i++ { + b = append(b, box("moof", make([]byte, 20))...) + } + b = append(b, bytes.Repeat([]byte{0xAB}, 3<<20)...) + b = append(b, box("free", nil)...) + err := structureOpen(t, b, "a.mp4", false) + require.Error(t, err) + assert.ErrorIs(t, err, errStructure) +} + +// A real fragmented tail still passes: moof+mdat chaining to the end (with an +// optional free box after), or an mfra whose mfro states its own size. +func TestForgery_RealFragmentTailsStillPass(t *testing.T) { + head := mp4Chain(mp4Box("ftyp", 16), mp4Box("moov", 300)) + for i := 0; i < 20; i++ { + head = append(head, mp4Box("moof", 40)...) + head = append(head, mp4Box("mdat", 200)...) + } + mfra := func() []byte { + mfro := box("mfro", []byte{0, 0, 0, 0, 0, 0, 0, 24}) // 16 bytes; mfra = 8 + 16 + return box("mfra", mfro) + } + for name, data := range map[string][]byte{ + "moof+mdat": head, + "moof+mdat then free": append(append([]byte{}, head...), mp4Box("free", 40)...), + "moof+mdat then mfra": append(append([]byte{}, head...), mfra()...), + "mfra then free, skip": append(append(append([]byte{}, head...), mfra()...), append(mp4Box("free", 10), mp4Box("skip", 12)...)...), + } { + t.Run(name, func(t *testing.T) { assert.NoError(t, structureOpen(t, data, "a.mp4", false)) }) + } + bad := append([]byte{}, head...) + bad = append(bad, box("mfra", box("mfro", []byte{0, 0, 0, 0, 0, 0, 0, 99}))...) // mfro lies about its mfra + assert.Error(t, structureOpen(t, bad, "a.mp4", false), "an mfro that does not describe its mfra") + lone := append(append([]byte{}, head[:len(head)-len(mp4Box("mdat", 200))]...), mp4Box("mdat", 200)...) + _ = lone + notPair := append(append([]byte{}, head...), mp4Box("sidx", 40)...) + assert.Error(t, structureOpen(t, notPair, "a.mp4", false), "a tail that is neither a moof+mdat pair nor an mfra") +} + +func TestStructure_NonFragmentedMP4WithManyTopLevelBoxes(t *testing.T) { + data := mp4Chain(mp4Box("ftyp", 16)) + for i := 0; i < 150; i++ { + data = append(data, mp4Box("free", 24)...) + } + data = append(data, mp4Box("moov", 200)...) + data = append(data, mp4Box("mdat", 600)...) + assert.NoError(t, structureOpen(t, data, "a.mp4", false), "a legitimate file with more than 63 boxes is not a 'too many reads' rejection") +} + +// Reviewer forgery: a Matroska with valid head (SeekHead promising Cues at the +// tail), a Cues ID at the promised place and nothing but zeros between. +func forgedMKV(total int, fill func(b []byte)) []byte { + segStartHdr := []byte{0x18, 0x53, 0x80, 0x67} + cuesPos := total - 4096 + seek := []byte{0x4D, 0xBB, 0x8E, 0x53, 0xAB, 0x84, 0x1C, 0x53, 0xBB, 0x6B, 0x53, 0xAC, 0x84, byte(cuesPos >> 24), byte(cuesPos >> 16), byte(cuesPos >> 8), byte(cuesPos)} + sh := append([]byte{0x11, 0x4D, 0x9B, 0x74, byte(0x80 | len(seek))}, seek...) + ebml := []byte{0x1A, 0x45, 0xDF, 0xA3, 0x84, 0x42, 0x82, 0x81, 0x00} + b := make([]byte, total) + if fill != nil { + fill(b) + } + copy(b, ebml) + off := len(ebml) + segLen := total - off - 4 - 8 + hdr := append(append([]byte{}, segStartHdr...), 0x01, byte(segLen>>48), byte(segLen>>40), byte(segLen>>32), byte(segLen>>24), byte(segLen>>16), byte(segLen>>8), byte(segLen)) + copy(b[off:], hdr) + segStart := off + len(hdr) + copy(b[segStart:], sh) + copy(b[segStart+cuesPos:], []byte{0x1C, 0x53, 0xBB, 0x6B}) + return b +} + +func TestForgery_MatroskaWithValidHeadAndTailButZeroMiddle(t *testing.T) { + err := structureOpen(t, forgedMKV(4<<20, nil), "a.mkv", false) + require.Error(t, err) + assert.ErrorIs(t, err, errStructure) +} + +// The same shape with real SimpleBlock chains in the middle is a real file. +func TestForgery_MatroskaWithRealClustersStillPasses(t *testing.T) { + data := forgedMKV(4<<20, func(b []byte) { fillSimpleBlocks(b, 200, 3000) }) + assert.NoError(t, structureOpen(t, data, "a.mkv", false)) +} + +func TestForgery_TransportStreamWithLostSyncInTheMiddle(t *testing.T) { + // 20000 packets; sync byte lost on a band of packets in the middle, which + // the fixed quartile positions can miss but random grid positions find. + ts := tsFile(188, 0, 20000) + for k := 3000; k < 17000; k++ { + if k%4 != 0 { // keep every fourth packet in sync, so a stride probe passes + ts[k*188] = 0 + } + } + hit := 0 + for range 40 { + if structureOpen(t, ts, "a.ts", false) != nil { + hit++ + } + } + assert.Greater(t, hit, 30, "random grid positions must find the broken band almost every time") +} + +// The open network only carries containers with a real structural check. +func TestOpenNetwork_GenericContainersAreNotVerifiable(t *testing.T) { + generic := make([]byte, 5000) + for i := range generic { + generic[i] = byte(i%251 + 1) + } + err := structureOpen(t, generic, "a.avi", true) + require.Error(t, err) + assert.Equal(t, ReasonContainerNotVerifiable, structureReason(err)) + assert.NoError(t, structureOpen(t, generic, "a.avi", false), "private networks keep the readable-only check") + assert.NoError(t, structureOpen(t, mkvPlain(4096), "a.mkv", true), "Matroska is verified, so open accepts it") + assert.NoError(t, structureOpen(t, tsFile(188, 0, 60), "a.ts", true)) + assert.Equal(t, ReasonContainerNotVerifiable, structureReason(structureOpen(t, append(make([]byte, 77), tsFile(188, 0, 60)...), "a.ts", true)), + "a capture that starts mid-packet is read as an unknown container") +} + +// simpleBlocks returns exactly n bytes (n >= 7) of consecutive Matroska +// SimpleBlock elements: what the body of a real Cluster looks like. +func simpleBlocks(n int) []byte { + out := make([]byte, 0, n) + for k := 0; len(out) < n; k++ { + remaining := n - len(out) + want := min(remaining, 90+(k*37)%300) + if remaining-want < 7 { + want = remaining + } + body := want - 3 // after the ID and the 2-byte size + out = append(out, 0xA3, 0x40|byte(body>>8), byte(body), 0x81, byte(k>>8), byte(k), 0x80) + for j := 0; j < body-4; j++ { + out = append(out, byte(1+(len(out)*7+k)%250)) + } + } + return out +} + +// fillSimpleBlocks writes a SimpleBlock chain over all of b. +func fillSimpleBlocks(b []byte, _, _ int) { copy(b, simpleBlocks(len(b))) } diff --git a/internal/peernet/bundleimport/structure_variants_test.go b/internal/peernet/bundleimport/structure_variants_test.go index e63a6e370..bafd65bee 100644 --- a/internal/peernet/bundleimport/structure_variants_test.go +++ b/internal/peernet/bundleimport/structure_variants_test.go @@ -22,7 +22,7 @@ func structureOf(t *testing.T, data []byte, name string) error { t.Helper() fs := afero.NewMemMapFs() require.NoError(t, afero.WriteFile(fs, name, data, 0o644)) - return verifyStructure(context.Background(), memOpener{fs}, name, int64(len(data))) + return verifyStructure(context.Background(), memOpener{fs}, name, int64(len(data)), false) } func requireAccepted(t *testing.T, data []byte, name string) { @@ -64,7 +64,7 @@ func mkvNoSeekHead(size int, live bool) []byte { out[len(out)-3] = byte(fill >> 16) out[len(out)-2] = byte(fill >> 8) out = append(out, byte(fill)) - out = append(out, bytes.Repeat([]byte{0x5A}, size-len(out))...) + out = append(out, simpleBlocks(size-len(out))...) return out } @@ -137,7 +137,8 @@ func mp4Fragmented(pairs int, lastMdat int, mfra bool) []byte { out = append(out, mp4Box("mdat", n)...) } if mfra { - out = append(out, mp4Box("mfra", 24)...) + // A real mfra: its only child is the mfro, which states the mfra's size. + out = append(out, box("mfra", box("mfro", []byte{0, 0, 0, 0, 0, 0, 0, 24}))...) } return out } diff --git a/internal/peernet/bundleimport/tracker.go b/internal/peernet/bundleimport/tracker.go index debae0939..50d855834 100644 --- a/internal/peernet/bundleimport/tracker.go +++ b/internal/peernet/bundleimport/tracker.go @@ -6,11 +6,40 @@ import ( "time" "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/peernet/contentid" ) // Tracker methods: the state machine of an accepted import. They are the // read and write side of "from the peer network, full check pending". +// isForeign reports whether the file at the row's path is another file than +// the one the row describes (a normal import or the user replaced it). A +// missing or unreadable file is not foreign: a rename may be in flight, and +// removing a missing file is harmless. Every action on a path (a delete, a +// hold, a forced check) first asks this; a foreign row is dropped. +func (i *Importer) isForeign(ctx context.Context, r database.PeernetBundleImport) bool { + if !i.cfg.Library.FileExists(r.FilePath) { + return false + } + m, err := i.cfg.Library.ReadFileMetadata(r.FilePath) + if err != nil || m == nil { + return false + } + id, err := contentid.FileID(m) + return err == nil && id.String() != r.FileID +} + +// dropForeign deletes the row of a foreign file and reports whether it did. +func (i *Importer) dropForeign(ctx context.Context, r database.PeernetBundleImport) bool { + if !i.isForeign(ctx, r) { + return false + } + slog.InfoContext(ctx, "peer import row dropped: the file at its path is not the imported one", "file_path", r.FilePath) + _ = i.cfg.State.Delete(context.WithoutCancel(ctx), []string{r.FilePath}) + i.releaseOrphanHolds(ctx) + return true +} + // PendingFullCheck returns which of paths are peer-bundle files whose full // check has not passed. The health worker forces a full check for them and // never gives them peer advice. It fails closed: on a database error every @@ -26,7 +55,10 @@ func (i *Importer) PendingFullCheck(ctx context.Context, paths []string) map[str return all } out := make(map[string]struct{}, len(rows)) - for p := range rows { + for p, r := range rows { + if i.dropForeign(ctx, r) { + continue + } out[p] = struct{}{} } return out @@ -39,7 +71,10 @@ func (i *Importer) IsPendingPeerImport(ctx context.Context, filePath string) boo return false } row, ok, err := i.cfg.State.Get(ctx, filePath) - return err == nil && ok && row.State == "pending" + if err != nil || !ok || row.State != "pending" { + return false + } + return !i.dropForeign(ctx, row) } // PeerBundleStatus is the read method the publisher uses: fromPeer says the @@ -77,6 +112,9 @@ func (i *Importer) QueueHasPending(ctx context.Context, queueID int64) bool { // FullCheckPassed records that a file's full check passed. When it was the // last file of its import, the ARRs are told, exactly once. func (i *Importer) FullCheckPassed(ctx context.Context, filePath string) { + if row, ok, err := i.cfg.State.Get(ctx, filePath); err == nil && ok && i.dropForeign(ctx, row) { + return + } won, err := i.cfg.State.MarkPassed(ctx, filePath) if err != nil || !won { if err != nil { @@ -94,7 +132,10 @@ func (i *Importer) FullCheckPassed(ctx context.Context, filePath string) { // No ARR is told about this import (the caller suppressed it, or it is a // direct import), so nothing else lifts the hold. if fh, ok := i.cfg.Announcer.(FailureHandler); ok && row.QueueID > 0 { - _ = fh.ReleaseHold(ctx, row.QueueID) + if err := fh.ReleaseHold(ctx, row.QueueID); err != nil { + // Housekeeping retries: SettledHolds finds this item again. + slog.WarnContext(ctx, "could not release the hold of a verified peer import; will retry", "queue_id", row.QueueID, "error", err) + } } } if v, ok := i.cfg.Reporter.(BundleVerifiedReporter); ok && !i.QueueHasPending(ctx, row.QueueID) { @@ -126,7 +167,7 @@ func (i *Importer) announce(ctx context.Context, queueID int64) { // release as having served a bad bundle. func (i *Importer) FullCheckFailed(ctx context.Context, filePath string) { row, ok, err := i.cfg.State.Get(ctx, filePath) - if err != nil || !ok { + if err != nil || !ok || i.dropForeign(ctx, row) { return } won, err := i.cfg.State.MarkFailed(ctx, filePath) @@ -134,10 +175,16 @@ func (i *Importer) FullCheckFailed(ctx context.Context, filePath string) { return } i.cnt.add(func(c *counters) { c.failed++ }) - i.failImport(ctx, row, failReasonCheck) - if i.cfg.Reporter == nil { + bundleID, _ := i.cfg.State.BundleIDOf(context.WithoutCancel(ctx), row.FilePath) // before the rows go + i.failImport(ctx, row, failReasonCheck, false) + // Only a contradiction between the layout and the articles' own headers + // (VerifyPeerHeaders) is the publisher's doing. A missing article shortly + // after posting can be propagation, and a degraded or unreadable file says + // nothing about the publisher, so none of those is reported. + if i.cfg.Reporter == nil || !i.takeMismatch(row.FilePath) { return } + i.noteFailedBundle(ctx, bundleID, row.ReleaseID) i.reportedMu.Lock() first := i.reported == nil || !i.reported[row.ReleaseID+"/"+row.PublisherID] if i.reported == nil || len(i.reported) >= maxReported { @@ -154,10 +201,31 @@ func (i *Importer) FullCheckFailed(ctx context.Context, filePath string) { i.cnt.add(func(c *counters) { c.reports++ }) } +// FullCheckDegraded ends a pending import whose full check found the media +// playable with a few missing segments. A release a peer vouched for must be +// whole, so it is failed at once (never announced, published or kept). When the +// user supplied the NZB (the fast path) the queue item is not failed: it is put +// back and imported normally, once, so the user gets what a normal import +// would have produced. The publisher is not reported (missing articles are not +// attributable). +func (i *Importer) FullCheckDegraded(ctx context.Context, filePath string) { + row, ok, err := i.cfg.State.Get(ctx, filePath) + if err != nil || !ok || i.dropForeign(ctx, row) { + return + } + won, err := i.cfg.State.MarkFailed(ctx, filePath) + if err != nil || !won { + return + } + i.cnt.add(func(c *counters) { c.failed++ }) + i.failImport(ctx, row, failReasonDegraded, true) +} + // Messages a failed peer import carries in the SABnzbd-compatible history. const ( - failReasonCheck = "Peer-supplied release failed its full health check and was removed" - failReasonStale = "Peer-supplied release was not fully verified within 7 days and was removed" + failReasonCheck = "Peer-supplied release failed its full health check and was removed" + failReasonStale = "Peer-supplied release was not fully verified within 7 days and was removed" + failReasonDegraded = "Peer-supplied release played with missing segments and was removed" ) // failImport ends a peer import that failed its full check, or never got one: @@ -168,7 +236,10 @@ const ( // does not stay in the library and the surviving files of the release cannot // complete the import by passing later. It touches only the rows of this // import, and every step is idempotent. -func (i *Importer) failImport(ctx context.Context, row database.PeernetBundleImport, reason string) { +// +// With requeue set (a degraded import) the queue item is put back for a normal +// import instead of failed, after the files are gone so the two cannot race. +func (i *Importer) failImport(ctx context.Context, row database.PeernetBundleImport, reason string, requeue bool) { bg := context.WithoutCancel(ctx) fh, _ := i.cfg.Announcer.(FailureHandler) rows := []database.PeernetBundleImport{row} @@ -176,7 +247,7 @@ func (i *Importer) failImport(ctx context.Context, row database.PeernetBundleImp if l, err := i.cfg.State.ListByQueue(bg, row.QueueID); err == nil && len(l) > 0 { rows = l } - if fh != nil { + if fh != nil && !requeue { if err := fh.FailImport(bg, row.QueueID, reason); err != nil { slog.WarnContext(ctx, "could not mark a failed peer import as failed in the queue", "queue_id", row.QueueID, "error", err) } @@ -185,6 +256,9 @@ func (i *Importer) failImport(ctx context.Context, row database.PeernetBundleImp paths := make([]string, 0, len(rows)) for _, r := range rows { paths = append(paths, r.FilePath) + if i.isForeign(ctx, r) { + continue // another file sits there now: never remove it + } var err error if fh != nil { err = fh.RemoveFailedFile(bg, r.FilePath) @@ -201,6 +275,18 @@ func (i *Importer) failImport(ctx context.Context, row database.PeernetBundleImp if err := i.cfg.State.Delete(bg, paths); err != nil { slog.WarnContext(ctx, "could not drop the state of a failed peer import", "error", err) } + if requeue && row.QueueID > 0 && fh != nil { + if rq, ok := fh.(Requeuer); ok { + if done, err := rq.RequeueForNormalImport(bg, row.QueueID); err != nil { + slog.WarnContext(ctx, "could not put a degraded peer import back for a normal import", "queue_id", row.QueueID, "error", err) + } else if done { + return + } + } + if err := fh.FailImport(bg, row.QueueID, reason); err != nil { + slog.WarnContext(ctx, "could not mark a failed peer import as failed in the queue", "queue_id", row.QueueID, "error", err) + } + } } // releaseOrphanHolds lifts the hold of every queue item that no state row @@ -216,9 +302,15 @@ func (i *Importer) releaseOrphanHolds(ctx context.Context) { slog.WarnContext(ctx, "could not list orphaned peer holds", "error", err) return } + // Holds of finished imports whose release failed once are retried here. + if settled, err := i.cfg.State.SettledHolds(ctx); err != nil { + slog.WarnContext(ctx, "could not list settled peer holds", "error", err) + } else { + ids = append(ids, settled...) + } for _, id := range ids { if err := fh.ReleaseHold(ctx, id); err != nil { - slog.WarnContext(ctx, "could not release an orphaned peer hold", "queue_id", id, "error", err) + slog.WarnContext(ctx, "could not release a peer hold", "queue_id", id, "error", err) } } } @@ -258,9 +350,12 @@ func (i *Importer) EscalateStale(ctx context.Context) int { } n := 0 for _, r := range rows { + if i.dropForeign(ctx, r) { + continue + } if won, err := i.cfg.State.MarkFailed(ctx, r.FilePath); err == nil && won { n++ - i.failImport(ctx, r, failReasonStale) + i.failImport(ctx, r, failReasonStale, false) } } if n > 0 { diff --git a/internal/peernet/bundleimport/tracker_test.go b/internal/peernet/bundleimport/tracker_test.go index 636b50799..b29fcf84f 100644 --- a/internal/peernet/bundleimport/tracker_test.go +++ b/internal/peernet/bundleimport/tracker_test.go @@ -112,7 +112,7 @@ type announcerFunc func(Announcement) error func (f announcerFunc) AnnounceVerifiedImport(_ context.Context, a Announcement) error { return f(a) } -func TestTracker_FailedFullCheckReportsPublisherAndNeverAnnounces(t *testing.T) { +func TestTracker_FailedFullCheckWithoutAMismatchDoesNotReportThePublisher(t *testing.T) { e := newEnv(t) b, res := e.accept("bad", mkvSpec("a.mkv"), mkvSpec("b.mkv")) ctx := context.Background() @@ -120,14 +120,15 @@ func TestTracker_FailedFullCheckReportsPublisherAndNeverAnnounces(t *testing.T) e.imp.FullCheckFailed(ctx, res.Paths[1]) e.imp.FullCheckFailed(ctx, res.Paths[0]) // passed already: no-op - assert.Equal(t, []string{strings.Repeat("d", 64) + "/" + b.ReleaseID.String()}, e.rep.got) + _ = b + assert.Empty(t, e.rep.got, "a missing-only failure may be propagation: the publisher is not reported") assert.Empty(t, e.ann.got) _, passed, _ := e.imp.PeerBundleStatus(ctx, res.Paths[1]) assert.False(t, passed, "a failed file never becomes publish-eligible") c := e.imp.Counters() assert.Equal(t, uint64(1), c.FullCheckFailed) assert.Equal(t, uint64(1), c.FullCheckPassed) - assert.Equal(t, uint64(1), c.BadBundleReports) + assert.Equal(t, uint64(0), c.BadBundleReports) } func TestTracker_CallerSuppressedARRIsNeverAnnounced(t *testing.T) { diff --git a/internal/peernet/bundleimport/types.go b/internal/peernet/bundleimport/types.go index 8d71207ef..02e6a4091 100644 --- a/internal/peernet/bundleimport/types.go +++ b/internal/peernet/bundleimport/types.go @@ -40,6 +40,9 @@ type Origin struct { // Corroborators is how many distinct network ranges announced this very // Bundle ID. Corroborators int + // BundleID is the Bundle ID (64 hex) the bundle was fetched under; it keys + // the negative cache of layouts that failed local verification. + BundleID string } // Fetcher returns a bundle that is already hash- and ID-verified (bundle.Verify) @@ -49,6 +52,30 @@ type Fetcher interface { FetchBundle(ctx context.Context, releaseID string) (*bundle.Bundle, Origin, error) } +// FetchOptions are the caller's requirements for the offers, applied before +// any peer is contacted. +type FetchOptions struct { + // AllowOpen says offers of the open network may be taken at all. + AllowOpen bool + // MinOpenCorroborators is how many distinct network ranges must announce an + // open-network offer's Bundle ID. + MinOpenCorroborators int +} + +// OptionFetcher is optionally implemented by a Fetcher: it filters the offers +// by the caller's requirements before dialing anything. +type OptionFetcher interface { + FetchBundleWith(ctx context.Context, releaseID string, o FetchOptions) (*bundle.Bundle, Origin, error) +} + +// BundleFailedReporter is optionally implemented by a BadBundleReporter: it is +// told that a Bundle ID's layout failed local verification (an attributable +// contradiction), so the same layout is skipped from every publisher for a +// while. +type BundleFailedReporter interface { + ReportFailedBundle(ctx context.Context, bundleID, releaseID string) +} + // BadBundleReporter is told when a publisher served a bundle whose release // failed its full local check, so the network side can back that publisher off. // It must return quickly and never fail. @@ -141,6 +168,9 @@ type State interface { ListByQueue(ctx context.Context, queueID int64) ([]database.PeernetBundleImport, error) ListAfter(ctx context.Context, afterPath string, limit int) ([]database.PeernetBundleImport, error) OrphanedHolds(ctx context.Context) ([]int64, error) + SetBundleID(ctx context.Context, paths []string, bundleID string) error + BundleIDOf(ctx context.Context, filePath string) (string, error) + SettledHolds(ctx context.Context) ([]int64, error) } // Announcement is what the ARRs are told about once every file of a bundle @@ -173,6 +203,14 @@ type FailureHandler interface { ReleaseHold(ctx context.Context, queueID int64) error } +// Requeuer is optionally implemented by the FailureHandler. It puts the held +// queue item of a fast-path import (one the user supplied an NZB for) back for +// a normal import, with the peer fast path off for that item once, and reports +// whether it did. A direct import has no NZB and is not requeueable (false). +type Requeuer interface { + RequeueForNormalImport(ctx context.Context, queueID int64) (bool, error) +} + // Recorder creates the queue and history entries of a direct import (one with // no NZB) so it looks like a normal import to the rest of the system. It // returns the queue id used to hold the ARR announcement. @@ -224,6 +262,14 @@ const ( ReasonStructureInvalid Reason = "structure_invalid" // media structure check failed beyond the first bytes ReasonForbiddenFile Reason = "forbidden_file" // a file a media server or the OS could act on ReasonSidecarTooLarge Reason = "sidecar_too_large" // a subtitle or nfo over its size cap + // ReasonLayoutNotAllowedOpen: the open network only carries plain layouts. + ReasonLayoutNotAllowedOpen Reason = "layout_not_allowed_open" + // ReasonContainerNotVerifiable: the open network only carries media whose + // container has a real structural check. + ReasonContainerNotVerifiable Reason = "container_not_verifiable" + // ReasonTooManyTables: the layout has more segment tables than the yEnc + // sampling budget can cover at the minimum rate. + ReasonTooManyTables Reason = "too_many_tables" ) // RejectError is the typed rejection Accept returns. @@ -263,11 +309,14 @@ type Policy struct { MediaProbeTimeout time.Duration // AcceptTimeout bounds the whole of Accept. AcceptTimeout time.Duration - // MaxYencSamples caps the articles whose yEnc headers are read, and - // YencRandomPerTable is how many random ones each segment table adds to the - // first and last. - MaxYencSamples int - YencRandomPerTable int + // MaxYencSamples caps the articles whose yEnc headers are read, MaxYencBytes + // the declared bytes those reads cost (the pool reads whole articles) and + // YencTimeout the time of the whole sampling. Every segment table gets its + // first, last and at least one random interior article; a layout whose + // tables do not fit the cap at that rate is rejected, never left unsampled. + MaxYencSamples int + MaxYencBytes int64 + YencTimeout time.Duration // MaxMediaFiles caps the media files a bundle may write (each is verified). MaxMediaFiles int // MaxSidecars caps the small allow-listed files (subtitles, nfo) written. @@ -280,17 +329,18 @@ type Policy struct { // DefaultPolicy returns the default local limits. func DefaultPolicy() Policy { return Policy{ - MaxTotalSize: int64(2) << 40, // 2 TiB - MaxFiles: 2_000, - MaxMediaProbes: 4, - MediaProbeTimeout: 30 * time.Second, - AcceptTimeout: 2 * time.Minute, - MaxYencSamples: 24, - YencRandomPerTable: 2, - MaxMediaFiles: 40, - MaxSidecars: 50, - MaxSubtitleBytes: 16 << 20, - MaxNfoBytes: 1 << 20, + MaxTotalSize: int64(2) << 40, // 2 TiB + MaxFiles: 2_000, + MaxMediaProbes: 4, + MediaProbeTimeout: 30 * time.Second, + AcceptTimeout: 2 * time.Minute, + MaxYencSamples: 600, + MaxYencBytes: 512 << 20, + YencTimeout: 90 * time.Second, + MaxMediaFiles: 40, + MaxSidecars: 50, + MaxSubtitleBytes: 16 << 20, + MaxNfoBytes: 1 << 20, } } @@ -345,6 +395,11 @@ type Request struct { // percent above decoded ones, so they only bound the bundle's declared // decoded sizes; they cannot prove them. ExpectedNZBSizes [][]int64 + // ExpectedNZBNames, when set with ExpectedNZB, is the subject file name of + // every NZB entry ("" when its subject has none). The first sampled + // article of a table must carry the same yEnc name as its NZB entry, which + // binds the table to that entry. + ExpectedNZBNames []string } // Result describes an accepted bundle. @@ -402,17 +457,23 @@ func (c *counters) snapshot() Counters { FullCheckFailed: c.failed, Announced: c.announce, BadBundleReports: c.reports} } -// YencSamples and YencRandom return the sampling bounds with defaults. +// YencSamples returns the sampling bound with its default. func (p Policy) YencSamples() int { return defaultInt(p.MaxYencSamples, DefaultPolicy().MaxYencSamples) } -func (p Policy) YencRandom() int { - if p.YencRandomPerTable < 0 { - return 0 +// YencBytes is the byte budget of the yEnc sampling. +func (p Policy) YencBytes() int64 { + if p.MaxYencBytes <= 0 { + return DefaultPolicy().MaxYencBytes } - if p.YencRandomPerTable == 0 { - return DefaultPolicy().YencRandomPerTable + return p.MaxYencBytes +} + +// YencTimeoutOrDefault is the time budget of the yEnc sampling. +func (p Policy) YencTimeoutOrDefault() time.Duration { + if p.YencTimeout <= 0 { + return DefaultPolicy().YencTimeout } - return p.YencRandomPerTable + return p.YencTimeout } diff --git a/internal/peernet/bundleimport/verify_headers.go b/internal/peernet/bundleimport/verify_headers.go new file mode 100644 index 000000000..771341c91 --- /dev/null +++ b/internal/peernet/bundleimport/verify_headers.go @@ -0,0 +1,135 @@ +package bundleimport + +import ( + "context" + "fmt" + "log/slog" + + "github.com/kipsilabs/altmount/internal/holes" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" +) + +// Header sample of the forced full check: 1 percent of a file's articles, at +// least 20 and at most 500 (never more than the file has). +const ( + headerSampleMin = 20 + headerSampleMax = 500 +) + +func headerSampleSize(total int) int { + return min(total, min(max(total/100, headerSampleMin), headerSampleMax)) +} + +// tablesOfMeta lists the segment tables a stored file reads: its own and each +// shared outer volume. +func tablesOfMeta(name string, m *metapb.FileMetadata) []sampleTable { + var out []sampleTable + add := func(label string, segs []*metapb.SegmentData) { + t := sampleTable{name: label} + for _, s := range segs { + if s == nil || s.Id == "" || holes.IsPlaceholderID(s.Id) { + return // a table with a hole cannot be described by one file + } + t.ids = append(t.ids, s.Id) + t.sizes = append(t.sizes, s.SegmentSize) + t.prefix = append(t.prefix, t.total) + t.total += s.SegmentSize + } + if len(t.ids) > 0 { + out = append(out, t) + } + } + add(name, m.SegmentData) + for j, o := range m.SharedOuterSources { + add(fmt.Sprintf("%s (volume %d)", name, j+1), o.Segments) + } + return out +} + +// VerifyPeerHeaders is the optional extra of the forced full check of a pending +// peer import (it implements health.PeerHeaderVerifier). The STAT-only check +// cannot see a size or offset lie, so a further random sample of the file's +// articles, proportional to its size, has its yEnc headers read and compared +// with the recorded layout (part size, part offset, table total). It returns +// nil when the sample agrees (or the file is not a pending peer import), an +// error wrapping health.ErrPeerLayoutMismatch when it contradicts the layout +// (attributable to the publisher), and another error when it could not be +// completed. This is sampling, not proof. +func (i *Importer) VerifyPeerHeaders(ctx context.Context, filePath string) error { + if i.cfg.State == nil || i.cfg.Headers == nil { + return nil + } + row, ok, err := i.cfg.State.Get(ctx, filePath) + if err != nil || !ok || row.State != "pending" || i.isForeign(ctx, row) { + return nil + } + m, err := i.cfg.Library.ReadFileMetadata(filePath) + if err != nil || m == nil { + return nil + } + tables := tablesOfMeta(filePath, m) + var total int + for _, t := range tables { + total += len(t.ids) + } + if total == 0 { + return nil + } + var plan []sample + var bytes int64 + picked := map[sample]struct{}{} + for want, tries := headerSampleSize(total), 0; len(plan) < want && tries < 20*want; tries++ { + g := randN(total) + ti := 0 + for g >= len(tables[ti].ids) { + g -= len(tables[ti].ids) + ti++ + } + s := sample{ti, g} + if _, dup := picked[s]; dup { + continue + } + if bytes+tables[ti].sizes[g] > i.cfg.Policy.YencBytes() { + break + } + picked[s] = struct{}{} + plan = append(plan, s) + bytes += tables[ti].sizes[g] + } + ctx, cancel := context.WithTimeout(ctx, i.cfg.Policy.YencTimeoutOrDefault()) + defer cancel() + verdicts := i.runSamples(ctx, tables, plan) + var inconclusive error + for _, v := range verdicts { + switch v.reason { + case "": + case ReasonYencMismatch: + i.mismatched.Store(filePath, struct{}{}) + slog.WarnContext(ctx, "peer import: article header contradicts the layout", "file_path", filePath, "detail", v.err) + return &LayoutMismatchError{Detail: v.err.Error()} + default: + if inconclusive == nil { + inconclusive = v.err + } + } + } + return inconclusive +} + +// takeMismatch reports whether a header sample contradicted the layout of the +// file, and forgets it. +func (i *Importer) takeMismatch(path string) bool { + _, ok := i.mismatched.LoadAndDelete(path) + return ok +} + +// LayoutMismatchError says the sampled articles contradict the recorded +// layout. The health worker recognises it by PeerLayoutMismatch. +type LayoutMismatchError struct{ Detail string } + +func (e *LayoutMismatchError) Error() string { + return "article headers contradict the imported layout: " + e.Detail +} + +// PeerLayoutMismatch marks the error for the health worker. +func (e *LayoutMismatchError) PeerLayoutMismatch() bool { return true } diff --git a/internal/peernet/bundleimport/yenc_test.go b/internal/peernet/bundleimport/yenc_test.go index c48b47d53..4e94522b9 100644 --- a/internal/peernet/bundleimport/yenc_test.go +++ b/internal/peernet/bundleimport/yenc_test.go @@ -9,6 +9,8 @@ import ( "github.com/javi11/nntppool/v5" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/kipsilabs/altmount/internal/peernet/bundle" ) func TestAccept_SampledYencHeadersMustAgreeWithTheLayout(t *testing.T) { @@ -53,7 +55,7 @@ func (c *countingHeaders) SegmentHeader(ctx context.Context, id string) (YencHea return c.SegmentHeaders.SegmentHeader(ctx, id) } -func TestAccept_YencSamplesAreBounded(t *testing.T) { +func TestAccept_YencBudgetTooSmallForTheTablesRejects(t *testing.T) { e := newEnv(t) files := []fileSpec{} for i := 0; i < 10; i++ { @@ -62,31 +64,154 @@ func TestAccept_YencSamplesAreBounded(t *testing.T) { b := e.mkBundle("cap", files...) ch := &countingHeaders{SegmentHeaders: e.imp.cfg.Headers} e.imp.cfg.Headers = ch - e.imp.cfg.Policy.MaxYencSamples = 6 + e.imp.cfg.Policy.MaxYencSamples = 6 // ten tables need at least 30 _, err := e.imp.Accept(context.Background(), e.req(b)) + require.Error(t, err) + assert.Equal(t, ReasonTooManyTables, ReasonOf(err)) + assert.Zero(t, ch.n.Load(), "nothing is read when the budget cannot cover the tables") + e.assertNothingBehind() +} + +func TestAccept_InteriorLieIsFound(t *testing.T) { + for _, idx := range []int{1, 2} { + e := newEnv(t) + b := e.mkBundle(fmt.Sprintf("in%d", idx), mkvSpec("movie.mkv")) + beh := e.fp.BehaviorFor(b.Pool[idx]) + beh.YEnc.PartBegin += 3 + e.fp.SetBehavior(b.Pool[idx], beh) + _, err := e.imp.Accept(context.Background(), e.req(b)) + require.Error(t, err) + assert.Equal(t, ReasonYencMismatch, ReasonOf(err), "interior article %d", idx) + e.assertNothingBehind() + } +} + +func TestAccept_FirstArticleNameMustMatchTheNZBEntry(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("nm", mkvSpec("movie.mkv")) + beh := e.fp.BehaviorFor(b.Pool[0]) + beh.YEnc.FileName = "a1b2c3.bin" + e.fp.SetBehavior(b.Pool[0], beh) + req := e.req(b) + req.ExpectedNZB = [][]string{b.Pool} + req.ExpectedNZBNames = []string{"other-entry.bin"} + _, err := e.imp.Accept(context.Background(), req) + require.Error(t, err) + assert.Equal(t, ReasonYencMismatch, ReasonOf(err), "%v", err) + assert.Empty(t, e.rep.got, "a disagreement with the local NZB is not the publisher's fault") + e.assertNothingBehind() + + req.ExpectedNZBNames = []string{"A1B2C3.bin"} // obfuscated names bind too + _, err = e.imp.Accept(context.Background(), req) require.NoError(t, err) - assert.EqualValues(t, 6, ch.n.Load()) } -func TestPlanSamples_ChoosesFirstLastAndUnpredictableMiddles(t *testing.T) { - tab := sampleTable{ids: make([]string, 200)} +func TestSubjectFilename(t *testing.T) { + assert.Equal(t, "My.File.rar", SubjectFilename(`[01/10] - "My.File.rar" yEnc (1/50)`)) + assert.Equal(t, "", SubjectFilename(`no quotes here`)) + assert.Equal(t, "", SubjectFilename(`"unterminated`)) +} + +func mkTable(n int, main bool) sampleTable { + t := sampleTable{ids: make([]string, n), sizes: make([]int64, n), prefix: make([]int64, n), main: main} + for i := range t.sizes { + t.sizes[i] = 100 + } + return t +} + +func TestPlanSamples_EveryTableGetsFirstLastAndAnInterior(t *testing.T) { + tables := make([]sampleTable, 50) + for i := range tables { + tables[i] = mkTable(10, false) + } + plan, err := planSamples(tables, 600, 1<<30) + require.NoError(t, err) + per := map[int][]int{} + for _, s := range plan { + per[s.table] = append(per[s.table], s.pos) + } + require.Len(t, per, 50, "no table is left unsampled") seen := map[int]struct{}{} - for range 40 { - plan := planSamples([]sampleTable{tab}, 2, 24) - require.Len(t, plan, 4) - assert.Equal(t, 0, plan[0].pos) - assert.Equal(t, 199, plan[1].pos) - for _, s := range plan[2:] { - assert.Greater(t, s.pos, 0) - assert.Less(t, s.pos, 199) - seen[s.pos] = struct{}{} + for _, ps := range per { + require.Len(t, ps, 3) + assert.Contains(t, ps, 0) + assert.Contains(t, ps, 9) + for _, p := range ps { + if p != 0 && p != 9 { + seen[p] = struct{}{} + } } } - assert.Greater(t, len(seen), 20, "middle positions must vary from call to call") + assert.Greater(t, len(seen), 3, "interior picks vary") +} + +func TestPlanSamples_MediaTablesAreSampledDenserAndTheCapHolds(t *testing.T) { + plan, err := planSamples([]sampleTable{mkTable(5000, true)}, 600, 1<<30) + require.NoError(t, err) + assert.Len(t, plan, 2+10, "first, last and one interior per 500 segments") + plan, err = planSamples([]sampleTable{mkTable(5000, true), mkTable(10, false)}, 8, 1<<30) + require.NoError(t, err) + assert.LessOrEqual(t, len(plan), 8) +} + +func TestPlanSamples_RejectsWhenTheMinimumDoesNotFit(t *testing.T) { + tables := make([]sampleTable, 20) + for i := range tables { + tables[i] = mkTable(10, false) + } + _, err := planSamples(tables, 59, 1<<30) + require.ErrorIs(t, err, errTooManyTables) + _, err = planSamples(tables, 600, 100) // bytes + require.ErrorIs(t, err, errTooManyTables) +} - many := make([]sampleTable, 50) - for i := range many { - many[i] = sampleTable{ids: make([]string, 10)} +// Two tables that start with the same article are two tables (they used to +// collapse into one when keyed by their first message-id). +func TestTablesOf_KeysByTheWholeTable(t *testing.T) { + seg := func(r int64) bundle.Segment { return bundle.Segment{Ref: r, Size: 10, Start: 0, End: 9} } + b := &bundle.Bundle{Pool: []string{"a@x", "b@x", "c@x"}, Files: []bundle.File{ + {Name: "A.mkv", Segments: []bundle.Segment{seg(0), seg(1)}}, + {Name: "B.mkv", Segments: []bundle.Segment{seg(0), seg(2)}}, + }} + assert.Len(t, tablesOf(b, nil), 2) +} + +// The cost of the accept-time header sampling for typical release shapes (768 KB +// articles). Logged for the documentation; the bounds are asserted. +func TestSamplingCostByReleaseShape(t *testing.T) { + pol := DefaultPolicy() + table := func(n int, main bool) sampleTable { + tb := mkTable(n, main) + for i := range tb.sizes { + tb.sizes[i] = 768_000 + } + return tb + } + volumes := func(n int) []sampleTable { + out := make([]sampleTable, n) + for i := range out { + out[i] = table(140, false) // a 100 MB volume + } + return out + } + for name, tc := range map[string]struct { + tables []sampleTable + wantMax int + }{ + "1 file (1.1 GB mkv)": {[]sampleTable{table(1500, true)}, 5}, + "50 volumes": {volumes(50), 150}, + "200 volumes": {volumes(200), 600}, + } { + plan, err := planSamples(tc.tables, pol.YencSamples(), pol.YencBytes()) + require.NoError(t, err, name) + var bytes int64 + for _, s := range plan { + bytes += tc.tables[s.table].sizes[s.pos] + } + t.Logf("%s: %d articles, %.0f MB read", name, len(plan), float64(bytes)/1e6) + assert.LessOrEqual(t, len(plan), tc.wantMax, name) } - assert.Len(t, planSamples(many, 2, 24), 24, "the total cap holds") + _, err := planSamples(volumes(201), pol.YencSamples(), pol.YencBytes()) + require.ErrorIs(t, err, errTooManyTables, "201 volumes exceed the budget and are rejected, not skipped") } diff --git a/internal/peernet/bundlex/buckets.go b/internal/peernet/bundlex/buckets.go index 29807bf52..451174453 100644 --- a/internal/peernet/bundlex/buckets.go +++ b/internal/peernet/bundlex/buckets.go @@ -54,6 +54,15 @@ func (b *buckets) take(key string, n float64, now time.Time) bool { return true } +// give returns n units to key's bucket (undoing a take). +func (b *buckets) give(key string, n float64) { + b.mu.Lock() + defer b.mu.Unlock() + if e := b.m[key]; e != nil { + e.tokens = min(b.burst, e.tokens+n) + } +} + // counter caps concurrent holders per key. type counter struct { mu sync.Mutex diff --git a/internal/peernet/bundlex/builder.go b/internal/peernet/bundlex/builder.go index 2781b16b1..839e0dab8 100644 --- a/internal/peernet/bundlex/builder.go +++ b/internal/peernet/bundlex/builder.go @@ -38,8 +38,13 @@ const ( // peer requests, node-wide. serveBuildsPerHour = 360 serveBuildBurst = 3 - negativeTTL = time.Minute - maxNegative = 256 + // serveBuildsPerRequesterPerHour is what one requester (a node ID, or a + // public address) may use of that: a sixth. Its burst is one, so no + // requester can spend more than a third of the node-wide burst. + serveBuildsPerRequesterPerHour = serveBuildsPerHour / 6 + serveBuildRequesterBurst = 1 + negativeTTL = time.Minute + maxNegative = 256 ) // NotShareableError says a release cannot be shared as a bundle, for a @@ -79,6 +84,8 @@ type Builder struct { // own publishing builds, and bound them node-wide whoever asks. serveSlot chan struct{} rate *buckets + byNode *buckets // per requesting node ID + byIP *buckets // per public requesting address neg map[string]negEntry mu sync.Mutex @@ -110,7 +117,9 @@ func NewBuilder(cfg BuilderConfig) *Builder { cfg.BuildWait = defaultBuildWait } return &Builder{cfg: cfg, slot: make(chan struct{}, 1), serveSlot: make(chan struct{}, 1), - rate: newBuckets(serveBuildsPerHour, serveBuildBurst, 1), neg: map[string]negEntry{}, cache: map[string]cacheEntry{}, + rate: newBuckets(serveBuildsPerHour, serveBuildBurst, 1), + byNode: newBuckets(serveBuildsPerRequesterPerHour, serveBuildRequesterBurst, maxRequesters), + byIP: newBuckets(serveBuildsPerRequesterPerHour, serveBuildRequesterBurst, maxRequesters), neg: map[string]negEntry{}, cache: map[string]cacheEntry{}, byBundle: map[bundle.BundleID]string{}, notShare: map[string]int64{}} } @@ -243,10 +252,63 @@ func (b *Builder) buildAndStore(ctx context.Context, releaseHex, title string, p return built, err } +// Requester identifies who asked for a build, so the node-wide build budget can +// be shared between requesters. +type Requester struct { + // Node is the authenticated node ID, IP the remote address (may be empty). + Node, IP string +} + +type requesterKey struct{} + +// WithRequester attaches the requester to ctx for BuildForServe. +func WithRequester(ctx context.Context, node, ip string) context.Context { + return context.WithValue(ctx, requesterKey{}, Requester{Node: node, IP: ip}) +} + +// maxRequesters bounds the per-requester accounting tables. +const maxRequesters = 4096 + +// spendServeBudget takes one build from the requester's node and address +// shares and from the node-wide budget, all or nothing. It is called only once +// the build slot is held and the build is about to start: a request that found +// the slot taken, timed out, was cancelled or hit the cache costs nothing. +func (b *Builder) spendServeBudget(ctx context.Context) bool { + now := b.cfg.Now() + req, _ := ctx.Value(requesterKey{}).(Requester) + type taken struct { + bk *buckets + key string + } + var got []taken + take := func(bk *buckets, key string) bool { + if !bk.take(key, 1, now) { + return false + } + got = append(got, taken{bk, key}) + return true + } + ok := true + if req.Node != "" { + ok = take(b.byNode, req.Node) + } + if ok && req.IP != "" && !sharedIP(req.IP) { // a LAN address is shared by honest nodes + ok = take(b.byIP, req.IP) + } + ok = ok && take(b.rate, "all") + if !ok { + for _, g := range got { + g.bk.give(g.key, 1) + } + } + return ok +} + // BuildForServe is Build for a request from a peer: it has its own build slot // (so a stream of peer requests can never starve this node's own publishing -// builds), a node-wide build rate independent of who asks, and remembers an -// unbuildable release for a short while. +// builds), a node-wide build rate that no single requester (node ID or public +// address) can use more than a sixth of, and remembers an unbuildable release +// for a short while. The budget is spent only when a build actually starts. func (b *Builder) BuildForServe(ctx context.Context, releaseHex, title string, posted time.Time) (*Built, error) { if c, ok := b.cached(releaseHex); ok { return c, nil @@ -254,9 +316,6 @@ func (b *Builder) BuildForServe(ctx context.Context, releaseHex, title string, p if err := b.negative(releaseHex); err != nil { return nil, err } - if !b.rate.take("all", 1, b.cfg.Now()) { - return nil, ErrBusy - } t := time.NewTimer(b.cfg.BuildWait) defer t.Stop() select { @@ -270,6 +329,12 @@ func (b *Builder) BuildForServe(ctx context.Context, releaseHex, title string, p if c, ok := b.cached(releaseHex); ok { return c, nil } + if err := ctx.Err(); err != nil { + return nil, err + } + if !b.spendServeBudget(ctx) { + return nil, ErrBusy + } built, err := b.buildAndStore(ctx, releaseHex, title, posted) var ns *NotShareableError if errors.As(err, &ns) { diff --git a/internal/peernet/bundlex/dos_test.go b/internal/peernet/bundlex/dos_test.go index c8237105b..438b20a67 100644 --- a/internal/peernet/bundlex/dos_test.go +++ b/internal/peernet/bundlex/dos_test.go @@ -94,3 +94,79 @@ func TestServeBuildsAreRateLimitedNodeWideAndUnbuildableIsRememberedBriefly(t *t t.Fatal("the second attempt must be answered from the negative cache") } } + +func distinctRelease(i int) string { + id := make([]byte, 32) + id[0], id[1] = byte(i), byte(i>>8) + out := "" + for _, c := range id { + out += string("0123456789abcdef"[c>>4]) + string("0123456789abcdef"[c&15]) + } + return out +} + +// A request that finds the build slot taken, times out or is cancelled never +// started a build, so it must not spend any of the node-wide budget. +func TestServeBuildBudgetIsSpentOnlyWhenABuildStarts(t *testing.T) { + lib, _ := nzbRelease() + b := NewBuilder(BuilderConfig{Source: lib, Library: lib, Patches: NoPatches{}, BuildWait: 10 * time.Millisecond}) + b.serveSlot <- struct{}{} // the slot is taken by a long build + for i := 0; i < 20; i++ { + ctx := WithRequester(context.Background(), "node-"+string(rune('a'+i)), "198.51.100."+string(rune('1'+i%9))) + if _, err := b.BuildForServe(ctx, distinctRelease(i), "T", posted); !errors.Is(err, ErrBusy) { + t.Fatalf("want ErrBusy while the slot is taken, got %v", err) + } + } + cctx, cancel := context.WithCancel(WithRequester(context.Background(), "node-z", "203.0.113.5")) + cancel() + _, _ = b.BuildForServe(cctx, distinctRelease(99), "T", posted) + <-b.serveSlot // the slot frees up + for i := 0; i < serveBuildBurst; i++ { + ctx := WithRequester(context.Background(), "node-"+string(rune('A'+i)), "203.0.113."+string(rune('1'+i))) + _, err := b.BuildForServe(ctx, distinctRelease(100+i), "T", posted) + if errors.Is(err, ErrBusy) { + t.Fatalf("build %d refused: the refused requests above spent the node-wide budget", i) + } + } +} + +// One requester (node, or public address) can use only a fraction of the +// node-wide budget, so a single identity cannot exhaust it for everyone. +func TestServeBuildBudgetIsSharedFairlyBetweenRequesters(t *testing.T) { + lib, _ := nzbRelease() + b := newTestBuilder(t, lib, NoPatches{}) + greedy := WithRequester(context.Background(), "node-greedy", "198.51.100.7") + ok := 0 + for i := 0; i < 20; i++ { + if _, err := b.BuildForServe(greedy, distinctRelease(i), "T", posted); !errors.Is(err, ErrBusy) { + ok++ + } + } + if ok == 0 || ok >= serveBuildBurst { + t.Fatalf("one requester built %d times; it must be limited below the node-wide burst of %d", ok, serveBuildBurst) + } + other := WithRequester(context.Background(), "node-other", "203.0.113.9") + if _, err := b.BuildForServe(other, distinctRelease(50), "T", posted); errors.Is(err, ErrBusy) { + t.Fatal("another requester must still get its share") + } + // A new identity behind the same public address shares that address's share. + sameIP := WithRequester(context.Background(), "node-fresh", "198.51.100.7") + if _, err := b.BuildForServe(sameIP, distinctRelease(60), "T", posted); !errors.Is(err, ErrBusy) { + t.Fatal("a fresh node ID at an exhausted public address must not get a fresh share") + } +} + +// A cache hit costs nothing. +func TestServeCacheHitsCostNoBudget(t *testing.T) { + lib, rel := nzbRelease() + b := newTestBuilder(t, lib, NoPatches{}) + ctx := WithRequester(context.Background(), "node-a", "198.51.100.1") + if _, err := b.BuildForServe(ctx, rel, "T", posted); err != nil { + t.Fatal(err) + } + for i := 0; i < 50; i++ { + if _, err := b.BuildForServe(ctx, rel, "T", posted); err != nil { + t.Fatalf("cache hit %d refused: %v", i, err) + } + } +} diff --git a/internal/peernet/bundlex/fetch.go b/internal/peernet/bundlex/fetch.go index d86fbee71..782991415 100644 --- a/internal/peernet/bundlex/fetch.go +++ b/internal/peernet/bundlex/fetch.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "sort" + "strings" "sync" "time" @@ -27,7 +28,10 @@ var ( ErrNoOffers = errors.New("bundlex: no fetchable offer for this release") ErrRateLimited = errors.New("bundlex: fetch rate limit reached") ErrFetchBusy = errors.New("bundlex: too many bundle downloads in progress") - ErrAllFailed = errors.New("bundlex: no offered bundle could be fetched and verified") + // ErrBackedOff: every fetchable offer is backed off. It is also an + // ErrNoOffers, so callers that do not tell them apart keep working. + ErrBackedOff = fmt.Errorf("%w: every offer is backed off", ErrNoOffers) + ErrAllFailed = errors.New("bundlex: no offered bundle could be fetched and verified") ) // Fetch failure reasons (counted in the status). @@ -61,6 +65,34 @@ type ReceivingNetworks interface { ReceivingNetworks() []recsync.NetworkInfo } +// FailedBundles is the negative cache of Bundle IDs whose layout failed local +// verification (nil: none). An offer for such a Bundle ID is skipped before any +// peer is contacted, whoever announces it. +type FailedBundles interface { + IsFailedBundle(ctx context.Context, bundleID string) bool +} + +// FetchOptions are the caller's requirements, applied to the offers BEFORE any +// peer is dialled (a contact reveals the Release ID, so an offer the caller +// would discard after fetching must never be contacted). The zero value +// restricts nothing. +type FetchOptions struct { + // Restrict switches the open-network rules below on. + Restrict bool + // AllowOpen says offers of the open network may be taken at all. + AllowOpen bool + // MinOpenCorroborators is how many distinct network ranges must announce an + // open-network offer's Bundle ID. + MinOpenCorroborators int +} + +func (o FetchOptions) permits(of Offer) bool { + if !o.Restrict || !of.Network.IsOpen() { + return true + } + return o.AllowOpen && of.Corroborators >= o.MinOpenCorroborators +} + // FetchConfig wires a Fetcher. type FetchConfig struct { Self identity.NodeID @@ -71,6 +103,8 @@ type FetchConfig struct { Auth discovery.Authorizer Limits FetchLimits Now func() time.Time + // Failed is the negative cache of failed Bundle IDs (nil: none). + Failed FailedBundles } // Offer is one stored release record that announces a bundle for a release. @@ -114,11 +148,12 @@ type Fetcher struct { pub *buckets slot chan struct{} - mu sync.Mutex - strikes map[string]*strike - ok int64 - failed map[string]int64 - refused int64 + mu sync.Mutex + refusedBundles map[bundle.BundleID]time.Time + strikes map[string]*strike + ok int64 + failed map[string]int64 + refused int64 // fcStrikes are the full-check penalties (see ReportBadBundle), kept apart // from the strikes of hostile bytes. @@ -197,6 +232,31 @@ func (f *Fetcher) backedOff(nw netid.NetworkID, p identity.NodeID) bool { return s != nil && s.until.After(now) } +// refuseBundle counts a plain refusal and skips that Bundle ID for SoftBackoff. +func (f *Fetcher) refuseBundle(id bundle.BundleID) { + f.mu.Lock() + defer f.mu.Unlock() + f.failed[failRefused]++ + if f.refusedBundles == nil { + f.refusedBundles = map[bundle.BundleID]time.Time{} + } + if len(f.refusedBundles) >= f.lim.MaxTracked { + f.refusedBundles = map[bundle.BundleID]time.Time{} + } + f.refusedBundles[id] = f.cfg.Now().Add(f.lim.SoftBackoff) +} + +func (f *Fetcher) bundleRefused(id bundle.BundleID) bool { + f.mu.Lock() + defer f.mu.Unlock() + until, ok := f.refusedBundles[id] + if ok && !f.cfg.Now().Before(until) { + delete(f.refusedBundles, id) + return false + } + return ok +} + func (f *Fetcher) fail(nw netid.NetworkID, p identity.NodeID, reason string, hostile bool) { f.mu.Lock() defer f.mu.Unlock() @@ -283,9 +343,23 @@ func (f *Fetcher) Offers(ctx context.Context, releaseHex string) ([]Offer, error if err != nil { return nil, err } + var attNodes []string for _, a := range atts { if a.Healthy { - set[a.PublisherNodeID] = true + attNodes = append(attNodes, a.PublisherNodeID) + } + } + // An attestation counts only from a known verified peer: anyone + // can sign one. + if len(attNodes) > 0 { + kn, err := f.cfg.Peers.Known(ctx, nw, attNodes) + if err != nil { + return nil, err + } + for _, a := range attNodes { + if kn[a] { + set[a] = true + } } } attesters[r.NetworkID] = set @@ -326,17 +400,15 @@ func (f *Fetcher) Offers(ctx context.Context, releaseHex string) ([]Offer, error return out, nil } -// rangeOf returns the network range of a peer address: /24 for IPv4, /48 for -// IPv6, the host itself for a name. -func rangeOf(addr string) string { - host, _, err := net.SplitHostPort(addr) - if err != nil { - host = addr - } - ip := net.ParseIP(host) +// rangeOf returns the network range of the IP this node actually connected to +// when it verified the peer: /24 for IPv4, /48 for IPv6, "" when none is +// recorded (such a peer counts for nothing). The stored address string is never +// used: a host name would let one server count as several ranges. +func rangeOf(connectedIP string) string { + ip := net.ParseIP(strings.TrimSpace(connectedIP)) switch { case ip == nil: - return "host:" + host + return "" case ip.To4() != nil: return ip.Mask(net.CIDRMask(24, 32)).String() + "/24" } @@ -362,7 +434,9 @@ func (f *Fetcher) corroborate(ctx context.Context, offers []Offer) error { if ranges[o.BundleID] == nil { ranges[o.BundleID] = map[string]struct{}{} } - ranges[o.BundleID][rangeOf(peer.Address)] = struct{}{} + if r := rangeOf(peer.ConnectedIP); r != "" { + ranges[o.BundleID][r] = struct{}{} + } } for i := range offers { offers[i].Corroborators = len(ranges[offers[i].BundleID]) @@ -401,6 +475,11 @@ func unhex(c byte) byte { // anything in the response. A publisher whose bytes fail verification is // backed off. Fetch never runs speculatively: only a local caller starts it. func (f *Fetcher) Fetch(ctx context.Context, releaseHex string) (*bundle.Bundle, FetchSource, error) { + return f.FetchWith(ctx, releaseHex, FetchOptions{}) +} + +// FetchWith is Fetch with the caller's requirements applied before dialing. +func (f *Fetcher) FetchWith(ctx context.Context, releaseHex string, opts FetchOptions) (*bundle.Bundle, FetchSource, error) { rid, err := contentid.ParseID(releaseHex) if err != nil { return nil, FetchSource{}, fmt.Errorf("bundlex: release id: %w", err) @@ -410,12 +489,24 @@ func (f *Fetcher) Fetch(ctx context.Context, releaseHex string) (*bundle.Bundle, return nil, FetchSource{}, err } var cands []Offer + backedOff := 0 for _, o := range offers { - if o.Fetchable && !f.backedOff(o.Network, o.Publisher) { - cands = append(cands, o) + if !o.Fetchable || !opts.permits(o) { + continue + } + if f.backedOff(o.Network, o.Publisher) || f.bundleRefused(o.BundleID) { + backedOff++ + continue + } + if f.cfg.Failed != nil && f.cfg.Failed.IsFailedBundle(ctx, o.BundleID.String()) { + continue } + cands = append(cands, o) } if len(cands) == 0 { + if backedOff > 0 { + return nil, FetchSource{}, ErrBackedOff + } return nil, FetchSource{}, ErrNoOffers } select { @@ -425,7 +516,7 @@ func (f *Fetcher) Fetch(ctx context.Context, releaseHex string) (*bundle.Bundle, return nil, FetchSource{}, ErrFetchBusy } var errs []error - attempts := 0 + attempts, pubLimited := 0, 0 for _, o := range cands { if attempts >= f.lim.MaxAttempts || ctx.Err() != nil { break @@ -441,6 +532,10 @@ func (f *Fetcher) Fetch(ctx context.Context, releaseHex string) (*bundle.Bundle, break } if !f.pub.take(strikeKey(o.Network, o.Publisher), 1, now) { + pubLimited++ + f.mu.Lock() + f.refused++ + f.mu.Unlock() continue } attempts++ @@ -457,6 +552,9 @@ func (f *Fetcher) Fetch(ctx context.Context, releaseHex string) (*bundle.Bundle, return nil, FetchSource{}, ctx.Err() } if len(errs) == 0 { + if pubLimited > 0 { + return nil, FetchSource{}, ErrRateLimited + } return nil, FetchSource{}, ErrNoOffers } return nil, FetchSource{}, fmt.Errorf("%w: %w", ErrAllFailed, errors.Join(errs...)) @@ -497,6 +595,15 @@ func (f *Fetcher) attempt(ctx context.Context, o Offer, rid contentid.ID) (*bund f.fail(o.Network, o.Publisher, reason, hostile) return nil, &fetchError{reason} } + if resp.StatusCode == http.StatusForbidden { + // The deliberately undecorated refusal (level lowered, release + // excluded, not a member) cannot be told apart by design: a plain + // refusal, never the publisher's fault. Only this bundle is skipped, and + // only briefly. + _ = resp.Body.Close() + f.refuseBundle(o.BundleID) + return nil, &fetchError{failRefused} + } if err := f.cfg.Auth.VerifyResponse(ctx, o.Network, o.Publisher, resp.StatusCode, resp.Header); err != nil { _ = resp.Body.Close() f.fail(o.Network, o.Publisher, failMembership, false) @@ -505,7 +612,7 @@ func (f *Fetcher) attempt(ctx context.Context, o Offer, rid contentid.ID) (*bund if resp.StatusCode != http.StatusOK { _ = resp.Body.Close() reason := failStatus - if resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests { + if resp.StatusCode == http.StatusTooManyRequests { reason = failRefused } f.fail(o.Network, o.Publisher, reason, false) @@ -520,7 +627,11 @@ func (f *Fetcher) attempt(ctx context.Context, o Offer, rid contentid.ID) (*bund f.fail(o.Network, o.Publisher, reason, hostile) return nil, &fetchError{reason} } - b, err := bundle.Verify(wire, o.BundleID, rid) + b, err := bundle.VerifyContext(ctx, wire, o.BundleID, rid) + if err != nil && ctx.Err() != nil { + // Waiting for the verification slot ran out of time: nobody's fault. + return nil, &fetchError{failTransport} + } if err != nil { f.fail(o.Network, o.Publisher, failVerification, true) return nil, &fetchError{failVerification} diff --git a/internal/peernet/bundlex/fetch_test.go b/internal/peernet/bundlex/fetch_test.go index c6e95c606..649dd1ad2 100644 --- a/internal/peernet/bundlex/fetch_test.go +++ b/internal/peernet/bundlex/fetch_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "net" "net/http" "sync" "testing" @@ -13,6 +14,7 @@ import ( metapb "github.com/kipsilabs/altmount/internal/metadata/proto" "github.com/kipsilabs/altmount/internal/peernet/bundle" "github.com/kipsilabs/altmount/internal/peernet/contentid" + "github.com/kipsilabs/altmount/internal/peernet/discovery" "github.com/kipsilabs/altmount/internal/peernet/identity" "github.com/kipsilabs/altmount/internal/peernet/netid" "github.com/kipsilabs/altmount/internal/peernet/recsync" @@ -365,7 +367,8 @@ func TestOffersPreferBundlesAnnouncedByMostDistinctNetworkRanges(t *testing.T) { mk := func() identity.NodeID { k, _ := identity.GenerateKey(); return k.NodeID() } announce := func(id identity.NodeID, addr, bundleID string, age time.Duration) { e.book.known[id.String()] = true - e.book.peers[id] = recsync.Peer{NodeID: id, Address: addr, Trusted: true} + host, _, _ := net.SplitHostPort(addr) + e.book.peers[id] = recsync.Peer{NodeID: id, Address: addr, Trusted: true, ConnectedIP: host} e.offers.rows = append(e.offers.rows, database.PeernetBundleOffer{NetworkID: nwA.String(), PublisherNodeID: id.String(), BundleID: bundleID, ReleaseScheme: database.PeernetSchemeNZB, CreatedAt: time.Now().Add(-age), Source: database.PeernetRecordSourcePeer}) } @@ -395,3 +398,207 @@ func TestOffersPreferBundlesAnnouncedByMostDistinctNetworkRanges(t *testing.T) { t.Fatalf("the corroborated bundle must rank first, got %v then %v", got[0].Publisher, got[1].Publisher) } } + +// Two DNS names for one server are one range, and a peer with no recorded +// connected IP counts for nothing. +func TestCorroborationKeysOnTheConnectedIPNotTheName(t *testing.T) { + e := newFetchEnv(t, FetchLimits{}) + mk := func() identity.NodeID { k, _ := identity.GenerateKey(); return k.NodeID() } + x := e.built.ID.String() + add := func(id identity.NodeID, addr, ip string) { + e.book.known[id.String()] = true + e.book.peers[id] = recsync.Peer{NodeID: id, Address: addr, Trusted: true, ConnectedIP: ip} + e.offers.rows = append(e.offers.rows, database.PeernetBundleOffer{NetworkID: nwA.String(), PublisherNodeID: id.String(), + BundleID: x, ReleaseScheme: database.PeernetSchemeNZB, CreatedAt: time.Now(), Source: database.PeernetRecordSourcePeer}) + } + a, b, c := mk(), mk(), mk() + add(a, "one.example.org:8470", "198.51.100.7") + add(b, "two.example.org:8470", "198.51.100.88") // same /24 under another name + add(c, "three.example.org:8470", "") // never recorded + got, err := e.f.Offers(context.Background(), e.rel) + if err != nil { + t.Fatal(err) + } + for _, o := range got { + if o.Corroborators != 1 { + t.Fatalf("%v: %d ranges, want 1 (two names, one /24; a peer without a connected IP counts nothing)", o.Publisher, o.Corroborators) + } + } +} + +// An attestation from a node that is not a known peer must not rank an offer. +func TestAttestersMustBeKnown(t *testing.T) { + e := newFetchEnv(t, FetchLimits{}) + mk := func() identity.NodeID { k, _ := identity.GenerateKey(); return k.NodeID() } + pub, stranger := mk(), mk() + e.book.known[pub.String()] = true + e.book.peers[pub] = recsync.Peer{NodeID: pub, Address: "10.1.1.1:8470", Trusted: true, ConnectedIP: "10.1.1.1"} + e.offers.rows = append(e.offers.rows, database.PeernetBundleOffer{NetworkID: nwA.String(), PublisherNodeID: pub.String(), + BundleID: e.built.ID.String(), ReleaseScheme: database.PeernetSchemeNZB, CreatedAt: time.Now(), Source: database.PeernetRecordSourcePeer}) + e.offers.atts = []database.PeernetAttestation{{PublisherNodeID: stranger.String(), Healthy: true}} + got, err := e.f.Offers(context.Background(), e.rel) + if err != nil || len(got) != 1 { + t.Fatalf("%v %v", got, err) + } + if got[0].AttestedHealthy { + t.Fatal("an unknown node's attestation must not count") + } + e.book.known[stranger.String()] = true + if got, _ = e.f.Offers(context.Background(), e.rel); !got[0].AttestedHealthy { + t.Fatal("control: a known attester counts") + } +} + +type openRecv struct{} + +func (openRecv) ReceivingNetworks() []recsync.NetworkInfo { + return []recsync.NetworkInfo{{ID: netid.Open(), Level: 3}} +} + +type failedSet map[string]bool + +func (f failedSet) IsFailedBundle(_ context.Context, id string) bool { return f[id] } + +// The caller's requirements filter the offers BEFORE anything is dialled: an +// open-network offer the caller will not take is never contacted (the contact +// itself reveals the Release ID). +func TestFetchWithFiltersOffersBeforeDialing(t *testing.T) { + e := newFetchEnv(t, FetchLimits{}) + e.f.cfg.Networks = openRecv{} + var hits int + var mu sync.Mutex + addr, id := hostile(t, func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + hits++ + mu.Unlock() + w.Header().Set("Content-Length", itoa(len(e.built.Wire))) + _, _ = w.Write(e.built.Wire) + }) + e.book.known[id.String()] = true + e.book.peers[id] = recsync.Peer{NodeID: id, Address: addr, Trusted: true, ConnectedIP: "127.0.0.1"} + e.offers.rows = append(e.offers.rows, database.PeernetBundleOffer{NetworkID: netid.Open().String(), PublisherNodeID: id.String(), + BundleID: e.built.ID.String(), ReleaseScheme: database.PeernetSchemeNZB, CreatedAt: time.Now(), Source: database.PeernetRecordSourcePeer}) + ctx := context.Background() + + for name, opts := range map[string]FetchOptions{ + "open not allowed": {Restrict: true}, + "one corroborator, two need": {Restrict: true, AllowOpen: true, MinOpenCorroborators: 2}, + } { + if _, _, err := e.f.FetchWith(ctx, e.rel, opts); !errors.Is(err, ErrNoOffers) { + t.Fatalf("%s: want ErrNoOffers, got %v", name, err) + } + mu.Lock() + n := hits + mu.Unlock() + if n != 0 { + t.Fatalf("%s: the peer was dialled %d times", name, n) + } + } + if _, _, err := e.f.FetchWith(ctx, e.rel, FetchOptions{Restrict: true, AllowOpen: true, MinOpenCorroborators: 1}); err != nil { + t.Fatalf("control: %v", err) + } +} + +// A Bundle ID that failed local verification is skipped, whoever offers it. +func TestFetchSkipsBundlesOnTheNegativeCache(t *testing.T) { + e := newFetchEnv(t, FetchLimits{}) + var hits int + var mu sync.Mutex + addr, id := hostile(t, func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + hits++ + mu.Unlock() + w.Header().Set("Content-Length", itoa(len(e.built.Wire))) + _, _ = w.Write(e.built.Wire) + }) + e.publisher(addr, id, time.Hour) + e.f.cfg.Failed = failedSet{e.built.ID.String(): true} + if _, _, err := e.f.Fetch(context.Background(), e.rel); !errors.Is(err, ErrNoOffers) { + t.Fatalf("want ErrNoOffers, got %v", err) + } + mu.Lock() + defer mu.Unlock() + if hits != 0 { + t.Fatalf("dialled %d times", hits) + } +} + +// A publisher that lowered its level or excluded the release answers with the +// deliberately undecorated 403, which a membership-checking authoriser cannot +// tell from a non-member. It must count as a plain refusal ("refused"), back +// off that bundle only for a short while, and never as the publisher's fault. +type undecoratedRefusalAuth struct{ discovery.AllowAll } + +func (undecoratedRefusalAuth) VerifyResponse(_ context.Context, _ netid.NetworkID, _ identity.NodeID, status int, _ http.Header) error { + if status == http.StatusForbidden { + return errors.New("response is not decorated") + } + return nil +} + +func TestFetchUndecoratedRefusalIsNeutralAndOnlyBacksOffThatBundle(t *testing.T) { + e := newFetchEnv(t, FetchLimits{}) + e.f.cfg.Auth = undecoratedRefusalAuth{} + refuse := true + var mu sync.Mutex + addr, id := hostile(t, func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + if refuse { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + w.Header().Set("Content-Length", itoa(len(e.built.Wire))) + _, _ = w.Write(e.built.Wire) + }) + e.publisher(addr, id, time.Hour) + now := time.Now() + e.f.cfg.Now = func() time.Time { return now } + if _, _, err := e.f.Fetch(context.Background(), e.rel); err == nil { + t.Fatal("the refusal must fail the fetch") + } + st := e.f.Status() + if st.Failed[failMembership] != 0 || st.Failed[failRefused] != 1 { + t.Fatalf("an undecorated refusal is 'refused', not 'membership': %+v", st.Failed) + } + if st.BackedOff != 0 { + t.Fatalf("the publisher must not be struck: %+v", st) + } + // The same bundle is skipped for the short soft back-off only. + if _, _, err := e.f.Fetch(context.Background(), e.rel); !errors.Is(err, ErrNoOffers) { + t.Fatalf("the refused bundle is skipped briefly: %v", err) + } + mu.Lock() + refuse = false + mu.Unlock() + now = now.Add(e.f.lim.SoftBackoff + time.Second) + if _, _, err := e.f.Fetch(context.Background(), e.rel); err != nil { + t.Fatalf("the publisher returned to serving: %v", err) + } +} + +// A per-publisher limit and a publisher back-off are not "no offer". +func TestFetchReportsPublisherLimitAndBackOffAsTheirOwnErrors(t *testing.T) { + e := newFetchEnv(t, FetchLimits{PerPublisherPerHour: 1, PerPublisherBurst: 1}) + addr, id := honest(t, e) + e.publisher(addr, id, time.Hour) + if _, _, err := e.f.Fetch(context.Background(), e.rel); err != nil { + t.Fatal(err) + } + _, _, err := e.f.Fetch(context.Background(), e.rel) + if !errors.Is(err, ErrRateLimited) { + t.Fatalf("a per-publisher limit must be ErrRateLimited, got %v", err) + } + if e.f.Status().RateLimited != 1 { + t.Fatalf("fetch_rate_limited stays %d", e.f.Status().RateLimited) + } + + e2 := newFetchEnv(t, FetchLimits{}) + addr2, id2 := honest(t, e2) + e2.publisher(addr2, id2, time.Hour) + e2.f.ReportBadBundle(nwA, id2) + _, _, err = e2.f.Fetch(context.Background(), e2.rel) + if !errors.Is(err, ErrBackedOff) || !errors.Is(err, ErrNoOffers) { + t.Fatalf("a backed-off publisher is ErrBackedOff (still an ErrNoOffers): %v", err) + } +} diff --git a/internal/peernet/bundlex/serve.go b/internal/peernet/bundlex/serve.go index 659d53bc0..0f5861209 100644 --- a/internal/peernet/bundlex/serve.go +++ b/internal/peernet/bundlex/serve.go @@ -217,7 +217,7 @@ func (s *Server) handle(w http.ResponseWriter, r *http.Request) { s.deny(w, "excluded") return } - built, err := s.cfg.Provider.Bundle(r.Context(), ann.ReleaseID) + built, err := s.cfg.Provider.Bundle(WithRequester(r.Context(), peer.String(), ip), ann.ReleaseID) switch { case errors.Is(err, ErrBusy): s.refuse("busy") diff --git a/internal/peernet/discovery/client.go b/internal/peernet/discovery/client.go index 1f56c1ba6..fa7d62faa 100644 --- a/internal/peernet/discovery/client.go +++ b/internal/peernet/discovery/client.go @@ -37,6 +37,9 @@ func (s *Service) call(ctx context.Context, nc NetworkConfig, addr string, node _ = resp.Body.Close() return nil, fmt.Errorf("discovery: verify response: %w", err) } + if resp.RemoteIP != "" { + s.connIPs.Store(node.String(), resp.RemoteIP) + } return resp, nil } diff --git a/internal/peernet/discovery/discovery_test.go b/internal/peernet/discovery/discovery_test.go index fe3fe2b92..e1dfd1466 100644 --- a/internal/peernet/discovery/discovery_test.go +++ b/internal/peernet/discovery/discovery_test.go @@ -563,3 +563,48 @@ func TestExtraSourceCandidatesAreValidatedAndCapped(t *testing.T) { } var _ = transport.ErrInvalidAddress + +// A verified contact records the IP the node really connected to; that, and +// not the stored address string, is what corroboration keys on. +func TestContactRecordsTheConnectedIP(t *testing.T) { + a, b := newNode(t, nodeOpts{}), newNode(t, nodeOpts{}) + a.seed(b.id, b.addr, database.PeernetSourceExchange, false) + if r := tick(a); r.Succeeded != 1 { + t.Fatalf("contact: %+v", r) + } + ip, err := a.store.ConnectedIP(context.Background(), testNet.String(), b.id.String()) + if err != nil { + t.Fatal(err) + } + if ip != "127.0.0.1" { + t.Fatalf("connected ip %q, want the loopback address the test server listens on", ip) + } +} + +// A peer that cannot be contacted shows why in the status, instead of an empty +// last error while it backs off. +func TestFailedContactSurfacesTheLastErrorInThePeerStatus(t *testing.T) { + a := newNode(t, nodeOpts{}) + ghost := randID(t) + a.seed(ghost, "127.0.0.1:1", database.PeernetSourceExchange, false) // nothing listens there + tick(a) + peers, err := a.svc.Peers(context.Background(), testNet) + if err != nil { + t.Fatal(err) + } + var got *PeerStatus + for i := range peers { + if peers[i].NodeID == ghost.String() { + got = &peers[i] + } + } + if got == nil || got.Failures == 0 { + t.Fatalf("the contact should have failed: %+v", peers) + } + if got.LastError == "" { + t.Fatal("the status must say why the peer is not connecting") + } + if len(got.LastError) > 200 { + t.Fatalf("error text is bounded: %d", len(got.LastError)) + } +} diff --git a/internal/peernet/discovery/service.go b/internal/peernet/discovery/service.go index d017ffff7..8db3891b5 100644 --- a/internal/peernet/discovery/service.go +++ b/internal/peernet/discovery/service.go @@ -88,12 +88,14 @@ type Service struct { auth Authorizer now func() time.Time - mu sync.Mutex - nets map[string]*netState - sources []Source - helloLim *helloLimiter - dials *dialBackBudget - acks *lru[helloAck] // advertised addresses a peer acknowledged, per (network, node) + mu sync.Mutex + nets map[string]*netState + sources []Source + helloLim *helloLimiter + dials *dialBackBudget + acks *lru[helloAck] // advertised addresses a peer acknowledged, per (network, node) + connIPs sync.Map // node id -> remote IP of its latest successful call + contactErrs sync.Map // network|node -> text of its last failed contact logMu sync.Mutex lastLogged map[string]string diff --git a/internal/peernet/discovery/status.go b/internal/peernet/discovery/status.go index 6b99955c6..c2956d002 100644 --- a/internal/peernet/discovery/status.go +++ b/internal/peernet/discovery/status.go @@ -65,6 +65,7 @@ func (s *Service) Peers(ctx context.Context, network netid.NetworkID) ([]PeerSta NodeID: p.NodeID, Address: p.Address, Source: statusSource(p.Source), Verified: p.Verified, FirstSeen: p.FirstSeen, LastSuccess: p.LastSuccess, Failures: p.Failures, NodeIDChanged: p.ChangedNodeID != "", ChangedNodeID: p.ChangedNodeID, + LastError: s.lastContactError(p.NetworkID, p.NodeID), }) } s.mu.Lock() @@ -125,3 +126,12 @@ func (s *Service) Counts(ctx context.Context, network netid.NetworkID) (Counts, s.mu.Unlock() return c, nil } + +// lastContactError is the text of the peer's last failed contact ("" after a +// success or when it never failed since start). +func (s *Service) lastContactError(network, node string) string { + if v, ok := s.contactErrs.Load(network + "|" + node); ok { + return v.(string) + } + return "" +} diff --git a/internal/peernet/discovery/tick.go b/internal/peernet/discovery/tick.go index a5d1b744c..601f51d4c 100644 --- a/internal/peernet/discovery/tick.go +++ b/internal/peernet/discovery/tick.go @@ -4,6 +4,7 @@ import ( "context" "errors" "sort" + "strings" "sync" "sync/atomic" "time" @@ -317,6 +318,11 @@ func (s *Service) contact(ctx context.Context, nc NetworkConfig, p database.Peer promote = addr } serr := s.cfg.Store.RecordSuccess(sctx, p.NetworkID, p.NodeID, promote, now, now.Add(s.limits.RefreshInterval)) + s.contactErrs.Delete(p.NetworkID + "|" + p.NodeID) + if ip, ok := s.connIPs.LoadAndDelete(p.NodeID); ok && serr == nil { + // The IP the verified handshake really reached, for corroboration. + serr = s.cfg.Store.RecordConnectedIP(sctx, p.NetworkID, p.NodeID, ip.(string)) + } s.logChange(ctx, s.problemKey("record success", nc), serr) return learned, serr } @@ -326,6 +332,14 @@ func (s *Service) contact(ctx context.Context, nc NetworkConfig, p database.Peer if p.Source == database.PeernetSourceUser && errors.Is(err, transport.ErrNodeIDMismatch) { s.flagChanged(sctx, nc, p) } + // Remember why, so the status can show it (the text is bounded and holds no + // secret: it names the address the user or the network gave us). + s.contactErrs.Store(p.NetworkID+"|"+p.NodeID, truncate(strings.Map(func(r rune) rune { + if r < 0x20 || r == 0x7f { + return ' ' + } + return r + }, err.Error()), 200)) _, ferr := s.cfg.Store.RecordFailure(sctx, p.NetworkID, p.NodeID, now, s.backoffFor(p)) s.logChange(ctx, s.problemKey("record failure", nc), ferr) return learned, err diff --git a/internal/peernet/recsync/exchange.go b/internal/peernet/recsync/exchange.go index b1c2649ac..da17fb2a2 100644 --- a/internal/peernet/recsync/exchange.go +++ b/internal/peernet/recsync/exchange.go @@ -52,6 +52,9 @@ type Peer struct { // Trusted marks an address from local configuration rather than learned // from a peer (see transport.CallOptions.TrustedAddress). Trusted bool + // ConnectedIP is the literal IP this node last connected to when it + // verified the peer ("" when none is recorded). Corroboration keys on it. + ConnectedIP string } // PeerSource lists the verified peers of a network, at most limit of them. diff --git a/internal/peernet/recsync/exchange_test.go b/internal/peernet/recsync/exchange_test.go index cda6b08b9..408713137 100644 --- a/internal/peernet/recsync/exchange_test.go +++ b/internal/peernet/recsync/exchange_test.go @@ -70,6 +70,14 @@ func TestPullStoresRecordsAndAdvancesCursor(t *testing.T) { if b.x.Status(ctx, nw).LastPull.IsZero() { t.Fatal("last successful pull not recorded") } + // After a restart the in-memory time is gone; the persisted cursors still + // say when records were pulled. + b.x.mu.Lock() + b.x.lastPull = map[string]time.Time{} + b.x.mu.Unlock() + if b.x.Status(ctx, nw).LastPull.IsZero() { + t.Fatal("the last pull must survive a restart") + } } func TestPullFollowsMoreFlagAcrossPages(t *testing.T) { diff --git a/internal/peernet/recsync/status.go b/internal/peernet/recsync/status.go index bc7b0f35a..b126cf54e 100644 --- a/internal/peernet/recsync/status.go +++ b/internal/peernet/recsync/status.go @@ -21,8 +21,8 @@ type NetworkStatus struct { Attestations RecordCounts // Publishers is the number of distinct publishers of stored records. Publishers int - // LastPull is the last time a pull from any peer succeeded (zero: never - // since start). + // LastPull is the last time a pull from any peer succeeded (zero: never; + // derived from the persisted sync cursors after a restart). LastPull time.Time // Stored, Duplicates and Rejected count ingest outcomes since start; // Rejected is keyed by reason name. @@ -43,6 +43,17 @@ func (x *Exchange) Status(ctx context.Context, nw netid.NetworkID) NetworkStatus x.mu.Lock() st.LastPull = x.lastPull[nw.String()] x.mu.Unlock() + if st.LastPull.IsZero() { + // After a restart the in-memory time is gone: the sync cursors, which + // are persisted, still say when records were last pulled. + if lp, ok := x.cfg.Cursors.(interface { + LastPullAt(ctx context.Context, networkID string) (time.Time, error) + }); ok { + if at, err := lp.LastPullAt(ctx, nw.String()); err == nil { + st.LastPull = at + } + } + } stats, err := x.cfg.Store.Stats(ctx, nw.String()) x.logChange(ctx, "record stats "+nw.String()[:8], err) if err != nil { diff --git a/internal/peernet/service/bundle_import_e2e_test.go b/internal/peernet/service/bundle_import_e2e_test.go index a76388783..020aa8de9 100644 --- a/internal/peernet/service/bundle_import_e2e_test.go +++ b/internal/peernet/service/bundle_import_e2e_test.go @@ -123,7 +123,7 @@ func mkvFile(size int) []byte { // A Cluster whose 8-byte size covers the filler. fill := size - len(out) - 12 - 9 out = append(out, 0x1F, 0x43, 0xB6, 0x75, 0x01, 0, byte(fill>>40), byte(fill>>32), byte(fill>>24), byte(fill>>16), byte(fill>>8), byte(fill)) - out = append(out, make([]byte, fill)...) + out = append(out, simpleBlocks(fill)...) // a real Cluster body return append(out, append(append([]byte{}, cues...), 0x84, 0, 0, 0, 0)...) } @@ -227,6 +227,16 @@ func TestBundleImportEndToEnd_FullCheckFailureBacksThePublisherOffForFetches(t * t.Fatal(err) } path := res.Paths[0] + // Only a contradiction between the layout and an article's own header is + // the publisher's doing (missing articles can be propagation): find one in + // the sampled headers, as the forced full check does. + id := "seg-005@news.test" + beh := r.pool.BehaviorFor(id) + beh.YEnc.PartBegin += 3 + r.pool.SetBehavior(id, beh) + if err := r.imp.VerifyPeerHeaders(ctx, path); err == nil { + t.Fatal("the header sample must contradict the layout") + } r.imp.FullCheckFailed(ctx, path) r.imp.FullCheckFailed(ctx, path) // reported once per release @@ -277,3 +287,22 @@ func TestBundleImportEndToEnd_ArticlesMissingOnTheReceiversProvidersRejectsAndLe t.Fatal("a rejected import must not be recorded or announced") } } + +// simpleBlocks returns exactly n bytes (n >= 7) of consecutive Matroska +// SimpleBlock elements, what the body of a real Cluster looks like. +func simpleBlocks(n int) []byte { + out := make([]byte, 0, n) + for k := 0; len(out) < n; k++ { + remaining := n - len(out) + want := min(remaining, 90+(k*37)%300) + if remaining-want < 7 { + want = remaining + } + body := want - 3 + out = append(out, 0xA3, 0x40|byte(body>>8), byte(body), 0x81, byte(k>>8), byte(k), 0x80) + for j := 0; j < body-4; j++ { + out = append(out, byte(1+(len(out)*7+k)%250)) + } + } + return out +} diff --git a/internal/peernet/service/bundles.go b/internal/peernet/service/bundles.go index 908873e9d..eeae1f659 100644 --- a/internal/peernet/service/bundles.go +++ b/internal/peernet/service/bundles.go @@ -177,13 +177,49 @@ func (p peerBook) VerifiedPeer(ctx context.Context, nw netid.NetworkID, node ide return recsync.Peer{}, false, err } trusted := row.Source == database.PeernetSourceUser || row.Source == database.PeernetSourceLocal - return recsync.Peer{NodeID: node, Address: row.Address, Trusted: trusted}, true, nil + ip, err := p.store.ConnectedIP(ctx, nw.String(), node.String()) + if err != nil { + return recsync.Peer{}, false, err + } + return recsync.Peer{NodeID: node, Address: row.Address, Trusted: trusted, ConnectedIP: ip}, true, nil } func (p peerBook) Known(ctx context.Context, nw netid.NetworkID, nodes []string) (map[string]bool, error) { return p.store.KnownAmong(ctx, nw.String(), nodes, knownPeerAge, time.Now()) } +// failedBundleBook adapts the bundle-import repository to bundlex.FailedBundles +// (the negative cache of layouts that failed local verification): entries count +// for failedBundleTTL. +type failedBundleBook struct { + repo *database.PeernetBundleImportRepository +} + +// failedBundleTTL is how long a failed Bundle ID is skipped. A layout is +// content-addressed: it does not get better, so a week is a bound on memory, +// not a belief that it might. +const failedBundleTTL = 7 * 24 * time.Hour + +func (b failedBundleBook) IsFailedBundle(ctx context.Context, bundleID string) bool { + if b.repo == nil { + return false + } + bad, err := b.repo.IsFailedBundle(ctx, bundleID, time.Now().Add(-failedBundleTTL)) + return err == nil && bad +} + +// NoteBundleFailed puts a Bundle ID whose layout failed local verification on +// the negative cache; the same layout offered by anyone is then skipped. +func (s *Service) NoteBundleFailed(bundleID, releaseID string) { + n := s.currentNode() + if n == nil || n.deps.BundleState == nil || bundleID == "" { + return + } + ctx := context.Background() + _ = n.deps.BundleState.RecordFailedBundle(ctx, bundleID, releaseID, time.Now()) + _ = n.deps.BundleState.PruneFailedBundles(ctx, time.Now().Add(-failedBundleTTL)) +} + // startBuilder creates the bundle builder and the fetch client. Without a // full-metadata source the builder is absent: level 3 then announces no // bundles. @@ -195,6 +231,7 @@ func (n *node) startBuilder() { n.fetcher = bundlex.NewFetcher(bundlex.FetchConfig{ Self: n.nodeID, Store: d.Records, Peers: peerBook{store: d.Peers, maxFailures: recsync.DefaultLimits().MaxPeerFailures}, Networks: n.rx, Dialer: n.client, Auth: authBridge{mem: n.mem, onRefused: n.noteRefused}, + Failed: failedBundleBook{repo: d.BundleState}, }) src := d.BundleSource if src == nil { @@ -261,6 +298,12 @@ func (s *Service) BundleOffers(ctx context.Context, releaseID string) ([]Offer, // limited and concurrency limited, and is never called speculatively by the // service itself. func (s *Service) FetchBundle(ctx context.Context, releaseID string) (*bundle.Bundle, FetchSource, error) { + return s.FetchBundleWith(ctx, releaseID, bundlex.FetchOptions{}) +} + +// FetchBundleWith is FetchBundle with the caller's requirements applied to the +// offers before any peer is contacted. +func (s *Service) FetchBundleWith(ctx context.Context, releaseID string, opts bundlex.FetchOptions) (*bundle.Bundle, FetchSource, error) { n, err := s.requireNode() if err != nil { return nil, FetchSource{}, err @@ -268,7 +311,7 @@ func (s *Service) FetchBundle(ctx context.Context, releaseID string) (*bundle.Bu if n.fetcher == nil { return nil, FetchSource{}, ErrBundlesUnavailable } - return n.fetcher.Fetch(ctx, releaseID) + return n.fetcher.FetchWith(ctx, releaseID, opts) } // NoteBundleFullCheck tells the fetcher how the full local check of a release diff --git a/internal/peernet/service/bundles_failed_test.go b/internal/peernet/service/bundles_failed_test.go new file mode 100644 index 000000000..d43381672 --- /dev/null +++ b/internal/peernet/service/bundles_failed_test.go @@ -0,0 +1,29 @@ +package service + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/kipsilabs/altmount/internal/database" +) + +func TestFailedBundleBookReadsTheNegativeCache(t *testing.T) { + db, err := database.NewDB(database.Config{DatabasePath: filepath.Join(t.TempDir(), "db")}) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + repo := database.NewPeernetBundleImportRepository(db.Connection(), database.DialectSQLite) + book := failedBundleBook{repo: repo} + ctx := context.Background() + id := "ab" + string(make([]byte, 0)) + "cdef" + assert.False(t, book.IsFailedBundle(ctx, id)) + require.NoError(t, repo.RecordFailedBundle(ctx, id, "rel", time.Now())) + assert.True(t, book.IsFailedBundle(ctx, id)) + require.NoError(t, repo.RecordFailedBundle(ctx, "old", "rel", time.Now().Add(-failedBundleTTL-time.Hour))) + assert.False(t, book.IsFailedBundle(ctx, "old"), "entries expire after the TTL") + assert.False(t, failedBundleBook{}.IsFailedBundle(ctx, id), "no repository: nothing is skipped") +} diff --git a/internal/peernet/service/catchup_test.go b/internal/peernet/service/catchup_test.go new file mode 100644 index 000000000..a1958fb6d --- /dev/null +++ b/internal/peernet/service/catchup_test.go @@ -0,0 +1,74 @@ +package service + +import ( + "context" + "testing" + + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/peernet/publish" +) + +// A file that became healthy while the network was below level 2 was skipped +// (not_publishing). When the level rises, a bounded catch-up pass publishes the +// release record of indexed, locally healthy releases that were never +// published. It never invents freshness: no attestation is published without a +// new real check. +func TestCatchUpPublishesHealthyIndexedReleasesNeverPublished(t *testing.T) { + lh := &fakeLocalHealth{statuses: map[string]string{pubPath: "healthy"}} + f := newPubInstance(t, level(2), func(d *Deps) { d.Health = lh }) + f.seedFile(t) + f.start(t) + awaitFirstUpkeep(t, f) + ctx := context.Background() + nw := openID.String() + + // No health notice was ever sent for this file: whatever publishes it is the + // catch-up pass of the first upkeep (or this explicit one). + f.node(t).pipe.catchUp(ctx, 10) + + if _, ok, _ := f.state.Get(ctx, nw, database.PeernetRecordKindRelease, pubRelease, ""); !ok { + t.Fatal("the healthy indexed release must be published by the catch-up pass") + } + if _, ok, _ := f.state.Get(ctx, nw, database.PeernetRecordKindAttestation, pubRelease, pubFile); ok { + t.Fatal("an attestation is never published without a new check") + } +} + +func TestCatchUpDoesNothingWhenNotLocallyHealthy(t *testing.T) { + lh := &fakeLocalHealth{statuses: map[string]string{pubPath: "corrupted"}} + f := newPubInstance(t, level(2), func(d *Deps) { d.Health = lh }) + f.seedFile(t) + f.start(t) + awaitFirstUpkeep(t, f) + ctx := context.Background() + f.node(t).pipe.catchUp(ctx, 10) + if _, ok, _ := f.state.Get(ctx, openID.String(), database.PeernetRecordKindRelease, pubRelease, ""); ok { + t.Fatal("a release whose file is not healthy must not be published") + } + if f.status(t).Publish.Skipped["not_healthy"] == 0 { + t.Fatalf("expected a not_healthy skip: %+v", f.status(t).Publish.Skipped) + } +} + +func TestLevelsRoseTriggersOnlyWhenANetworkStartsPublishing(t *testing.T) { + var nets []publish.Network + p := &pipeline{cfg: pipelineConfig{Networks: func() []publish.Network { return nets }}} + if p.levelsRose() { + t.Fatal("the first pass after start is not a level change") + } + nets = []publish.Network{{ID: openID, Level: 2}} + if !p.levelsRose() { + t.Fatal("a network that started publishing must trigger a catch-up") + } + if p.levelsRose() { + t.Fatal("no change, no catch-up") + } + nets = []publish.Network{{ID: openID, Level: 3}} + if !p.levelsRose() { + t.Fatal("a rise from 2 to 3 triggers one") + } + nets = []publish.Network{{ID: openID, Level: 2}} + if p.levelsRose() { + t.Fatal("a drop does not") + } +} diff --git a/internal/peernet/service/publishing.go b/internal/peernet/service/publishing.go index 73e123a7d..3225390ad 100644 --- a/internal/peernet/service/publishing.go +++ b/internal/peernet/service/publishing.go @@ -302,6 +302,10 @@ type pipeline struct { backMu sync.Mutex backoff map[string]backoffEntry // network|release + + levelMu sync.Mutex + seenLevels map[string]int // network -> level at the last upkeep pass + catchUpMore atomic.Bool // a catch-up pass left releases for the next one } func newPipeline(cfg pipelineConfig) (*pipeline, error) { @@ -739,6 +743,12 @@ func (n *node) maintainPublishing(ctx context.Context, orphans bool) { } n.pipe.reannounce(ctx, reannounceBatch) n.pipe.reannounceLevelChanges(ctx, reannounceBatch) + // The catch-up scan of the content index runs when a network's level rose + // to publishing, while an earlier pass found more than one batch, and on the + // orphan cadence (not on the very first pass after start); never on every pass. + if n.pipe.levelsRose() || (orphans && n.maintRuns.Load() > 0) || n.pipe.catchUpMore.Swap(false) { + n.pipe.catchUp(ctx, reannounceBatch) + } } // retractExcluded deletes this node's own records and publish state of @@ -807,3 +817,69 @@ func (p *pipeline) reannounceLevelChanges(ctx context.Context, limit int) { } } } + +// levelsRose records the publishing networks' levels and reports whether any +// network started publishing (level 2 or higher) or rose since the last call. +func (p *pipeline) levelsRose() bool { + p.levelMu.Lock() + defer p.levelMu.Unlock() + seen := p.seenLevels + p.seenLevels = map[string]int{} + rose := false + for _, nw := range p.cfg.Networks() { + k := nw.ID.String() + p.seenLevels[k] = nw.Level + if old, ok := seen[k]; !ok || nw.Level > old { + rose = true + } + } + return rose && seen != nil // the first pass after start is not a level change +} + +// catchUp publishes, up to limit releases per network, the release records of +// indexed releases that were never published on a network that publishes now: +// files that became healthy while the network was below level 2 were skipped +// (not_publishing) and would otherwise wait for their next health check. Each +// release goes through the re-announce guard (every indexed file locally +// healthy, a peer import verified, exclusions applied). Attestations are never +// touched: they come only from a new real check. A release that cannot be +// published backs off like any re-announce, so it does not take later slots. +func (p *pipeline) catchUp(ctx context.Context, limit int) { + now := p.cfg.Now() + for _, nw := range p.cfg.Networks() { + if ctx.Err() != nil { + return + } + ids, err := p.cfg.State.ListUnpublishedReleases(ctx, nw.ID.String(), min(limit+p.backoffCount(now), reannounceScanMax)) + if err != nil { + p.cfg.Log(ctx, "publish: list releases to catch up", err) + return + } + p.cfg.Log(ctx, "publish: list releases to catch up", nil) + done := 0 + for _, id := range ids { + if ctx.Err() != nil { + return + } + rel, err := contentid.ParseID(id) + if err != nil || p.backedOff(nw.ID, id, now) { + continue + } + if done >= limit { + p.catchUpMore.Store(true) + break + } + done++ + t := publish.Tracked{Network: nw.ID, Kind: records.KindRelease, ReleaseID: records.ReleaseID(rel)} + if p.reannounceOne(ctx, t, id) { + p.clearFailure(nw.ID, id) + // A first publication, not a refresh: keep the re-announce counter honest. + p.stats.mu.Lock() + p.stats.reannounced-- + p.stats.mu.Unlock() + } else { + p.noteFailure(nw.ID, id, now) + } + } + } +} diff --git a/internal/peernet/service/service.go b/internal/peernet/service/service.go index 64a2e1815..9620955df 100644 --- a/internal/peernet/service/service.go +++ b/internal/peernet/service/service.go @@ -46,6 +46,10 @@ type Deps struct { // ever published. Exclusions is read on every publish, so config changes // apply at once; nil means none. Backbones may be nil: no attestation is // then published. + // BundleState is the bundle-import repository; its negative cache of failed + // Bundle IDs lets the fetcher skip a layout that failed local verification. + // Nil: no negative cache. + BundleState *database.PeernetBundleImportRepository PublishState *database.PeernetPublishStateRepository Origins *database.PeernetOriginRepository Exclusions func() publish.Exclusions diff --git a/internal/peernet/transport/client.go b/internal/peernet/transport/client.go index 766b6f086..a0f29533e 100644 --- a/internal/peernet/transport/client.go +++ b/internal/peernet/transport/client.go @@ -10,9 +10,11 @@ import ( "io" "net" "net/http" + "net/http/httptrace" "net/netip" "net/url" "sync" + "sync/atomic" "syscall" "time" @@ -88,6 +90,9 @@ type Request struct { // call's timer. Reading past CallOptions.MaxResponseBytes returns // ErrBodyTooLarge. type Response struct { + // RemoteIP is the literal IP the connection was made to ("" if unknown). + // A peer's corroboration is keyed on it, never on the address string. + RemoteIP string StatusCode int Header http.Header Body io.ReadCloser @@ -185,7 +190,16 @@ func (c *Client) Do(ctx context.Context, addr string, expected identity.NodeID, if len(req.Body) > 0 { body = bytes.NewReader(req.Body) } - hr, err := http.NewRequestWithContext(callCtx, req.Method, u.String(), body) + var remoteIP atomic.Value + trace := &httptrace.ClientTrace{GotConn: func(i httptrace.GotConnInfo) { + if i.Conn == nil { + return + } + if host, _, err := net.SplitHostPort(i.Conn.RemoteAddr().String()); err == nil { + remoteIP.Store(host) + } + }} + hr, err := http.NewRequestWithContext(httptrace.WithClientTrace(callCtx, trace), req.Method, u.String(), body) if err != nil { cancel() return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err) @@ -207,8 +221,9 @@ func (c *Client) Do(ctx context.Context, addr string, expected identity.NodeID, cancel() return nil, ErrBodyTooLarge } + ip, _ := remoteIP.Load().(string) return &Response{ - StatusCode: resp.StatusCode, Header: resp.Header, + RemoteIP: ip, StatusCode: resp.StatusCode, Header: resp.Header, Body: &cappedBody{rc: resp.Body, max: opts.MaxResponseBytes, onClose: cancel}, }, nil } From fee2a179bdf1bdf886ea6158cda2386b026e3f2d Mon Sep 17 00:00:00 2001 From: javi11 Date: Sat, 14 Nov 2026 20:16:37 +0100 Subject: [PATCH 3/5] feat(peernet): accept every layout on all networks with archive header checks Removes the restriction that limited the open network to plain layouts in structurally verifiable containers. Archive, keyed and clipped layouts and all media containers are accepted on every network again. - archive extents are bound to the archive's own file headers (RAR4, RAR5, 7z with a plain end header): stored entry, data start, packed size and continuation flags must match the bundle; a contradiction is attributed to the publisher - where headers cannot be read the file is accepted only with a real structural media check, never on readability alone - the article sampling budget scales with release size instead of rejecting releases with many volumes; the last article of every table is always checked - bounded header parsing with fuzz targets - documentation for the wiring guard, fetch outcomes, catch-up publish and the network terminology --- docs/docs/3. Configuration/peer-network.md | 14 +- .../specs/2026-11-12-peer-network-design.md | 23 + .../components/config/peernet/peernetStats.ts | 11 +- frontend/tests/peernetStats.test.ts | 15 +- internal/peernet/archivehdr/archivehdr.go | 343 +++++++++++++ .../peernet/archivehdr/archivehdr_test.go | 294 ++++++++++++ .../archivehdr/archivetest/archivetest.go | 272 +++++++++++ internal/peernet/archivehdr/crc_test.go | 13 + internal/peernet/archivehdr/fuzz_test.go | 75 +++ internal/peernet/archivehdr/rar.go | 323 +++++++++++++ internal/peernet/archivehdr/sevenzip.go | 452 ++++++++++++++++++ internal/peernet/bundleimport/accept.go | 82 ++-- internal/peernet/bundleimport/accept_test.go | 15 +- internal/peernet/bundleimport/cost.go | 56 +++ internal/peernet/bundleimport/extents.go | 401 ++++++++++++++++ internal/peernet/bundleimport/extents_test.go | 441 +++++++++++++++++ internal/peernet/bundleimport/sample.go | 213 +++++++-- internal/peernet/bundleimport/structure.go | 16 +- .../bundleimport/structure_forgery_test.go | 39 +- internal/peernet/bundleimport/types.go | 29 +- .../peernet/bundleimport/verify_headers.go | 4 +- internal/peernet/bundleimport/yenc_test.go | 193 ++++++-- .../peernet/service/status_levels_test.go | 35 ++ 23 files changed, 3166 insertions(+), 193 deletions(-) create mode 100644 internal/peernet/archivehdr/archivehdr.go create mode 100644 internal/peernet/archivehdr/archivehdr_test.go create mode 100644 internal/peernet/archivehdr/archivetest/archivetest.go create mode 100644 internal/peernet/archivehdr/crc_test.go create mode 100644 internal/peernet/archivehdr/fuzz_test.go create mode 100644 internal/peernet/archivehdr/rar.go create mode 100644 internal/peernet/archivehdr/sevenzip.go create mode 100644 internal/peernet/bundleimport/cost.go create mode 100644 internal/peernet/bundleimport/extents.go create mode 100644 internal/peernet/bundleimport/extents_test.go create mode 100644 internal/peernet/service/status_levels_test.go diff --git a/docs/docs/3. Configuration/peer-network.md b/docs/docs/3. Configuration/peer-network.md index a8560757b..cb8afc918 100644 --- a/docs/docs/3. Configuration/peer-network.md +++ b/docs/docs/3. Configuration/peer-network.md @@ -8,6 +8,8 @@ keywords: [altmount, peer network, peernet, sharing, privacy] The peer network lets AltMount instances help each other. An instance that has imported a release and found it healthy can tell others, so they can pick healthy releases, shorten their own health checks, or reuse a finished import. Media content is never transferred; it always comes from your own Usenet providers. Everything received from a peer is advisory and verified locally before use. +In this page the **open network** is the network of peers you add yourself plus the peers they know about: there is no public directory, no DHT and no built-in bootstrap node, so a fresh install knows nobody until you enter an address. + The feature is **off by default**. While it is off nothing listens, nothing connects and no key is generated. All settings below apply live; no restart is needed. ## Configuration @@ -175,14 +177,16 @@ Before anything is written the layout is also checked against the articles thems - **No gaps.** A bundle file may not contain a hole; a release with known holes is not shareable and a received bundle with one is refused. - **Coverage.** In a bundle for an NZB, every file (and every archive volume or PAR2 file) must be exactly one NZB file entry's segments, in NZB order; no segment is used twice and none is left out. - **Sizes against your NZB.** When you import an NZB, each declared decoded segment size must fit the NZB's (encoded, a few percent larger) size. -- **Sampled article headers.** For each file and archive volume the first and last article and a few random ones (chosen locally, up to 24 in all) are read from your providers, and their yEnc headers must agree with the layout: decoded size, start offset, and total file size. A mismatch refuses the bundle and counts against the publisher. The cost is at most 24 article downloads (roughly 18 MB) per import. +- **Sampled article headers.** Articles of each file and archive volume are read from your providers (which ones is described under "Sampling" below) and their yEnc headers must agree with the layout: decoded size, start offset, and total file size. A mismatch refuses the bundle and counts against the publisher. - **Media structure.** Besides the start of each media file, its container index is read at several positions including the end: Matroska/WebM (EBML header, Segment size, and every SeekHead entry such as the Cues), MP4/MOV (the whole top-level box chain must end exactly at the end of the file, or, for fragmented files with thousands of boxes, a fragment box must end exactly there), MPEG-TS/M2TS (sync bytes on the 188, 192 or 204 byte grid at the start, quarters and end). Files muxed for streaming that have no Matroska SeekHead, or an unknown-size Segment, are accepted when a level-1 element opens the Segment, a known Segment size lands on the end of the file and the quarters and the end can be read; trailing free, skip or uuid boxes, a zero-sized last box, 64-bit box sizes and up to 64 KiB of zero padding in MP4 are accepted; other containers get full-length reads at the quarters and end. Every written media file passes this, not only the largest. - **File types.** Only media files and small subtitle (`.srt .ass .sub .idx .sup`) and `.nfo` files are written (at most 40 media files and 50 sidecars, subtitles up to 16 MB, `.nfo` up to 1 MB). Files your *allowed file extensions* exclude are skipped like in a normal import. Playlists, links, queue files, scripts and executables (`.strm .nzb .nzbz .meta .url .lnk .m3u .m3u8 .pls .desktop .exe .sh ...`) make the whole bundle refused. - **Folder name.** The release folder comes from the title with the media extension removed first; names that are empty, hidden (start with a dot), reserved by AltMount or equal to a category folder are refused, so a title can never point outside the category folder. - **Offers.** Of several offers, the bundle announced identically by publishers in the most distinct network ranges is preferred; a publisher's own attestation never raises the rank of its own offer. -- **Plain layouts only from the open network.** A bundle offered through the open network is accepted only when every file is exactly one plain run of articles: archive volumes (nested layouts), encryption keys and multi-clip files are refused with `layout_not_allowed_open`, and media whose container has no structural check (anything but Matroska/WebM, MP4/MOV and MPEG-TS/M2TS) with `container_not_verifiable`. This is a stated limitation, not a bug: those layouts cannot be bound to the articles without reading archive headers, which is planned. Private networks still accept them. -- **Sampling.** Every run of articles is sampled: its first and last article and at least one random one in between (one per 500 for a media file, at least three) have their yEnc headers (part size, offset, total) compared with the layout, up to 600 articles and 512 MiB; a release with more archive volumes than that is refused (`too_many_tables`). A single movie file reads about 5 articles (4 MB), a 50-volume release 150 (115 MB), a 200-volume release 600 (461 MB). With your NZB at hand the first article's yEnc file name must also match the NZB entry it belongs to. The forced full check reads a further random sample of headers (1 percent of the articles, 20 to 500) and fails the import on a contradiction. It samples; it does not prove. +- **Every layout, every network.** Archive layouts (outer volumes read through extents), AES-keyed files, multi-clip files and every media container are accepted from the open network exactly as from a private one, provided they pass the checks below. There is no `layout_not_allowed_open` or `container_not_verifiable` rejection any more. Generic containers (AVI, MPEG and the like) get the head signature and readable-windows check wherever the bundle came from. +- **Archive extent checks.** For every file read through outer archive volumes, each extent (outer volume, offset, length) is compared with the archive's own file header, read from your providers through the same hidden reader that will serve the file (so an encrypted outer volume is read decrypted with the bundle's key). The header that precedes the extent's data must be a stored (not compressed) file entry, not a service block or a directory and not itself encrypted; its data must start exactly at the extent's offset and be as long as the extent in that volume; the entry's unpacked size must equal the file's size; and its continuation flags must fit the extent's place (first part, middle part, last part). RAR 4, RAR 5 and 7z (plain end header) are understood. When the outer volume is itself a stored file of a wrapping archive and the volume begins in that archive's first article, the wrapping header is checked as well (stored, same size, and for RAR 5 the IV in the header must equal the bundle's). Up to 64 extents are checked (the first and last of the main media file always, the rest chosen at random). A contradiction is attributable to the publisher: the bundle is refused (`extent_mismatch`), the publisher is reported (fetch back-off only) and the layout goes on the negative cache for 7 days. +- **What stays unverifiable.** An extent whose archive cannot be judged is *unverifiable*: an unknown format (a disc image such as a Blu-ray ISO), a 7z end header that is compressed or encrypted, encrypted archive headers, a header walk that exceeds its budget, a damaged header chain, or a missing article. Such a file is accepted only when its media passes the structural check (Matroska, MP4 or transport stream) through the layout; a generic container in that situation is refused (`extent_unverifiable`), because readable windows prove nothing about misplaced bytes. Extents beyond the 64 checked are covered by sampling only. Files stored inline in a single archive volume (no outer volumes) are not extents and are verified by the head and structure checks only. +- **Sampling.** The last article of every run of articles (volume or file) is always read: a size lie anywhere in a run shifts every later offset, so the last article's offset and total expose it. The main media file's runs also get their first article and random interior ones (one per 500 articles, at least three). The first and a random interior article of the other runs are read as far as the budget allows, at randomly chosen runs. The budget is one percent of the release size, at least 64 MiB and at most 1 GiB, shared with the archive header reads above and raised if the last articles alone need more; only a release whose last articles exceed 1 GiB (about 1,400 volumes) is refused (`too_many_tables`). Optional samples that run out of the time limit are dropped, mandatory ones are not. A single movie file reads about 5 articles (4 MB), a 50-volume release about 90 (67 MB), a 200-volume release about 280 (215 MB), a 500-volume release about 700 (540 MB), plus a few articles per checked archive volume. Two lies that cancel out inside a run are found only when a sampled article lies between them. With your NZB at hand the first article's yEnc file name must also match the NZB entry it belongs to. The forced full check reads a further random sample of headers (1 percent of the articles, 20 to 500) and fails the import on a contradiction. It samples; it does not prove. - **Media structure.** Besides the end of the file, the container is read at random positions inside the file (Matroska: three random 1 MiB windows must contain cluster data; MPEG-TS: eight random packet positions must be in sync; fragmented MP4 must end in a real `moof`+`mdat` pair or `mfra`). What remains unprovable: bytes inside MP4 `mdat`, inside a Matroska frame or between sampled positions. - **Degraded means failed.** A peer release whose full check finds the media playable but with missing segments is failed and removed, never announced. When you supplied the NZB the queue item is not marked failed: it is imported normally from your NZB, once, so you get what a normal import would have produced. A release imported directly (Stremio peer play) is failed. A missing article alone does not count against the publisher (it can be propagation just after posting); only a contradiction between the layout and the articles' own headers does, and that layout is then ignored whoever offers it, for 7 days. - **Corroboration** counts distinct /24 (IPv4) or /48 (IPv6) ranges of the address this node actually connected to when it last verified the publisher, never host names; with the open-network switch off, open-network publishers are not even contacted. @@ -199,6 +203,10 @@ Playing one asks your node (not the player) to fetch the bundle, verify it as de - Bundles: 8 MB compressed, 16 MB decompressed (about 340,000 segments of a typical release, roughly 250 GB), 500,000 segments and 10,000 files at most; a receiving node also refuses releases over 2 TiB or 2,000 files. - Fetching: on demand only, never speculative; at most 3 attempts of 60 seconds per request, 60 attempts per hour overall, 12 per publisher, 2 downloads at once. +- Startup guard: the bundle importer refuses to be built when a required collaborator (library, stager, prober, state, index, fetcher, header reader, recorder, announcer, reporter, health switch) is missing, so a misconfiguration fails at start instead of rejecting every bundle later. +- Fetch outcomes are counted by reason: the peer declined (`refused`), answered with an error status, a connection problem, a truncated or oversized download, a failed hash or ID verification, a scheme mismatch, or a membership problem. A serving node counts its own refusals as well (level, denied caller, backed off, rate limited, busy, unknown or excluded release, byte budget). Rate-limited fetches and publishers backed off after a bad bundle are shown in the status page; the back-off only affects bundle fetches, never record sync. +- Catch-up publish: when you raise a network's level so that it starts publishing (for example from 1 to 2, or 2 to 3), the next upkeep pass publishes the records of already-imported healthy releases once, instead of waiting for each file's next health check. +- The status page shows the time of the last successful record pull per network and keeps it across restarts, and the last contact error of each peer (an unreachable address, a changed node ID, a refused request). - The health check must be enabled to receive bundles, because the full check must be able to run. - A bundle is for the exact release: two NZBs of the same movie with different article sets are different releases. diff --git a/docs/superpowers/specs/2026-11-12-peer-network-design.md b/docs/superpowers/specs/2026-11-12-peer-network-design.md index 2554091c2..889765bbb 100644 --- a/docs/superpowers/specs/2026-11-12-peer-network-design.md +++ b/docs/superpowers/specs/2026-11-12-peer-network-design.md @@ -996,3 +996,26 @@ A second review with working PoCs found that the identities bind less than the f **State and failure.** Every action on a row's path (failure, hold, forced check, stale escalation) first compares the File ID at the path with the row; a foreign row is dropped and causes no delete, hold or forced check. A delete followed by a rewrite of the same File ID at the same path ends the peer import cleanly instead of re-attaching to a normal import. A degraded verdict fails a pending peer import at once (never announced, removed, no PAR2 repair queued); on the fast path the queue item is put back for a normal import with the peer fast path skipped once (`RequeueNormalImport`), a direct import is failed. A check that failed outright and ended corrupted counts like a corrupted verdict. The publisher is reported only for a header contradiction, not for missing articles (which can be propagation just after posting), and then the layout goes on a negative cache keyed by Bundle ID (`peernet_failed_bundles`, 7 days, persisted because failures surface up to a week later) that the fetcher consults before dialing. Housekeeping releases holds of finished imports owing no ARR (`SettledHolds`) in addition to orphaned ones, and `Reconcile` takes the write lock exclusively so it can never roll back a write in flight. Deleting a held queue item by download ID or path removes its provisional history. Idempotent `SafeFolderName`, the dead `Options.FilesReleaseID` hook, and the prebuilt NZB store reused after `TrimArticles` were fixed. **Residual risk.** Content between sampled points is not proven to be the titled media. That is the same class of risk as a mislabelled NZB from an indexer, now bounded by random sampling per table, header samples during the full check, structural checks at random interior positions, plain layouts only in the open network, and a fallback to a normal import when a peer layout does not hold up. + +## Phase 3: third hardening round + +Owner decision: every kind of release is accepted on every network. The second round's open-network restrictions (`layout_not_allowed_open`, `container_not_verifiable`) are removed; what they stood in for is now checked, and what cannot be checked is stated. + +**Archive-extent spot checks (every network).** A file read through outer volumes is described by extents (outer, InnerOffset, InnerLength) that no identifier binds. The importer produces them in two cases only: an inner RAR volume stored in an encrypted outer RAR or 7z archive (the outer source is the decrypted inner volume, the extent the inner file's data area), and expanded disc images (extents into the image). Plain stored RAR and 7z files, and AES-keyed ones, are flattened into one inline table with a partial first and last segment and have no extents. +- Mechanism (`internal/peernet/archivehdr`, used by `bundleimport/extents.go`). Each outer volume is staged as a file of its own (with the bundle's key and IV when keyed) and read through the staged reader, so nothing is visible to the user. `archivehdr.Scan` identifies RAR 5, RAR 4 or 7z by signature and walks the headers: RAR volumes from their start, hopping over data areas by the declared size; 7z by its plain end header. `Check` then requires: a block whose data area starts exactly at InnerOffset; a file entry (not a service block, not a directory); stored (RAR 5 method 0, RAR 4 method 0x30, 7z Copy coder); not itself encrypted; packed size equal to InnerLength; unpacked size equal to the file's size; split-before and split-after flags consistent with the extent being first, middle or last (7z: a single extent). The reusable library header reader (`rardecode`) cannot be used: it exposes neither continuation flags nor per-volume bounds, and it follows volume names. The parsers are small and strict instead. +- Wrapper check. When the outer table's first segment starts inside its article (an inner volume sliced out of an outer archive), the table is also read raw from the start of its first article; if that article starts a RAR archive, the entry whose data begins at the table's first byte must be stored, as large as the table, and (RAR 5 file encryption record) carry the bundle's IV. RAR 4 keeps its IV derived from the password and cannot be compared. A wrong key or IV shows as garbage where the inner volume's signature should be. +- Bounds. Per scan: 64 blocks, 160 reads, 64 KiB per header, 64 KiB 7z end header; per volume at most 24 distinct articles touched; every touched article is charged (declared size) to the accept-wide budget; at most 64 extents are checked (all of them up to that, else the first and last extent of the main media file plus a crypto/rand choice). No size from the archive is used to allocate; every offset is added with overflow checks; no recursion. Fuzz targets `FuzzVerify` and `FuzzVerifyUnchecked` (checksums off, to reach the parsers below them) run over seed archives built by `archivetest`. +- Outcomes. `Bound`; `Unverifiable` (unknown format, encoded 7z header, encrypted archive headers, budget spent, damaged header chain, read error); `Mismatch`. A mismatch is attributable: `extent_mismatch`, fetch-domain report, negative cache. A missing article is a probe error, never a mismatch. +- Accepted without header binding, always only with a structural media check through the layout (a generic container is refused, `extent_unverifiable`): disc-image extents; 7z with an encoded end header; encrypted RAR headers (not shareable on the publisher side anyway); volumes whose walk exceeded the budget; every file whose outer volumes are not a RAR or 7z archive. Extents beyond the 64 checked are covered by sampling only. Inline single-archive files (no extents) are accepted on the head and structure checks, as before. +- Found while studying the producer, not changed: a stored multi-volume RAR or 7z file with no outer encryption is flattened into one inline table with partial ranges at every volume boundary, which `Validate` rejects (`ErrPartialRange`), so such releases are not shareable today; and the yEnc whole-file check (offset zero, table total equals the file total) is only right for tables that cover a posted file from its first to its last article, so a table that is a slice from the middle of a larger posted file is refused by it. + +**One budget.** The yEnc header samples and the archive header reads share one budget of declared article bytes (`costBudget`). Order: mandatory samples, archive header checks, then optional samples with what is left. +- Mandatory: the last article of every table (a size lie anywhere in a table shifts every later offset, so the last article's offset and the table total expose a single lie with certainty); for the largest media file's tables also the first article and random interior ones (one per 500, at least three). +- Optional: the first article, then one random interior article, of other tables, in random order (crypto/rand) while the budget lasts. +- Cap: 1 percent of the declared release size, at least 64 MiB, at most 1 GiB (`MaxYencBytes`), raised to fit the mandatory samples and the expected header reads; the release is refused (`too_many_tables`) only when those exceed 1 GiB or 2000 articles. Optional samples that run out of the 90 second limit are dropped; mandatory ones are not. +- Cost at 768 KB articles: 1 file (1.1 GB) 5 articles, 4 MB; 50 volumes of 100 MB 87 articles, 67 MB; 200 volumes 280 articles, 215 MB; 300 volumes of 50 MB 300 articles, 230 MB (the last articles alone); 500 volumes 700 articles, 538 MB. Archive header reads for nested volumes cost one article per checked volume (two when the volume is a sliced inner volume with a wrapper check): 50 volumes 38 to 77 MB, 200 volumes (64 checked) 49 to 98 MB. +- Detection: a single size lie in a table is caught with certainty by the last article (mandatory for every table). Two compensating lies inside one table are caught only when a sampled interior or first article lies between them: for the main file about 3 per 1,500 articles at the minimum, for other tables 0 to 2 per table depending on the budget. + +**State and reasons.** Removed reasons: `layout_not_allowed_open`, `container_not_verifiable`. New: `extent_mismatch`, `extent_unverifiable`. `too_many_tables` now means the ceiling is exceeded. + +**Activity card.** On a node with the open network at level 0 and a private network at level 3 the status carries `level: 0` for the open network (from the configuration) and `level: 3` for the private one (from its row), both under the same field name, and the card's rules (`some(level >= 1)`, `some(level >= 3)`) evaluate correctly on that shape; a test pins it in the service and in the frontend. No defect in the derivation was found; the non-zero-counter fallback stays, because the serving counters are since-start counters and stay non-zero after a level is lowered. diff --git a/frontend/src/components/config/peernet/peernetStats.ts b/frontend/src/components/config/peernet/peernetStats.ts index 59ae25e06..9852c3d96 100644 --- a/frontend/src/components/config/peernet/peernetStats.ts +++ b/frontend/src/components/config/peernet/peernetStats.ts @@ -113,15 +113,12 @@ const IMPORT_REJECT_LABELS: ReadonlyMap = new Map([ ["size_implausible", "Declared file sizes do not fit your NZB"], ["yenc_mismatch", "Sampled articles contradict the bundle"], ["structure_invalid", "Media file structure is broken"], + ["extent_mismatch", "An archive part contradicts the archive's own file header"], [ - "layout_not_allowed_open", - "Archive or encrypted layouts are not accepted from the open network", + "extent_unverifiable", + "Archive layout could not be checked and the media format cannot stand in", ], - [ - "container_not_verifiable", - "Media container cannot be checked, so the open network does not carry it", - ], - ["too_many_tables", "More archive volumes than the verification budget covers"], + ["too_many_tables", "Too many archive volumes for the verification limit"], ]); export function notShareableLabel(key: string): string { diff --git a/frontend/tests/peernetStats.test.ts b/frontend/tests/peernetStats.test.ts index 62977e8c2..a1b93d034 100644 --- a/frontend/tests/peernetStats.test.ts +++ b/frontend/tests/peernetStats.test.ts @@ -138,8 +138,8 @@ const BACKEND_IMPORT_REJECT_KEYS = [ "size_implausible", "yenc_mismatch", "structure_invalid", - "layout_not_allowed_open", - "container_not_verifiable", + "extent_mismatch", + "extent_unverifiable", "too_many_tables", ]; @@ -175,6 +175,17 @@ describe("bundle block visibility", () => { expect(showBundleServing([{ level: 2 }], { serve_refused: { busy: 2 } })).toBe(true); expect(showBundleServing([{ level: 2 }], { announced: 0, fetched: 4 })).toBe(false); }); + test("open network at level 0 with a private network at level 3, as the API sends it", () => { + // The shape of GET /peernet/status in the live run: the open network's level comes + // from the configuration, the private network's from its own row; both use "level". + const status = JSON.parse( + '{"networks":[{"kind":"open","role":"open","level":0},{"kind":"private","role":"host","level":3}],' + + '"bundles":{"announced":0,"served":0,"import":{}}}', + ) as { networks: { level: number }[]; bundles: unknown }; + expect(showBundlesBlock(status.networks)).toBe(true); + expect(showBundleServing(status.networks, status.bundles)).toBe(true); + expect(showBundleServing(status.networks)).toBe(true); + }); test("activity detection ignores junk", () => { expect(bundleHasActivity(undefined)).toBe(false); expect(bundleHasActivity({ fetched: Number.NaN })).toBe(false); diff --git a/internal/peernet/archivehdr/archivehdr.go b/internal/peernet/archivehdr/archivehdr.go new file mode 100644 index 000000000..a8a88ee92 --- /dev/null +++ b/internal/peernet/archivehdr/archivehdr.go @@ -0,0 +1,343 @@ +// Package archivehdr reads the clear headers of an archive volume (RAR 4, +// RAR 5, 7z) far enough to say where the data area of each stored file starts, +// how long it is and whether it continues across volumes. It exists to bind a +// bundle's archive extents (InnerOffset, InnerLength) to the archive's own file +// headers. +// +// Every byte it reads is hostile: the archive was posted by anyone, and a +// bundle only claims to describe it. The parsers therefore +// +// - read through an io.ReaderAt with a hard cap on the number of reads, +// - never allocate from a declared size (a header is read into a buffer of at +// most Limits.MaxHeader bytes, a table of declared length is only decoded +// when the bytes it needs are inside the header already read), +// - use checked arithmetic for every offset and length, +// - walk a fixed number of blocks, iteratively (no recursion), and +// - never trust a CRC they have not verified. +// +// RAR volumes are walked from their start, hopping over data areas by their +// declared size. 7z keeps its headers at the end, so only a plain (not +// compressed, not encrypted) end header can be read; an encoded one is reported +// as ErrCompressedHeader and the caller treats the extent as unverifiable. +package archivehdr + +import ( + "errors" + "fmt" + "io" + "math" +) + +// Format is the archive format of a volume. +type Format uint8 + +// Formats. +const ( + FormatUnknown Format = iota + FormatRAR4 + FormatRAR5 + Format7z +) + +func (f Format) String() string { + switch f { + case FormatRAR4: + return "rar4" + case FormatRAR5: + return "rar5" + case Format7z: + return "7z" + } + return "unknown" +} + +// Errors that say a volume could not be judged (as opposed to Mismatch, which +// is a positive contradiction). +var ( + // ErrUnknownFormat: the volume does not start with a known archive signature. + ErrUnknownFormat = errors.New("archivehdr: not a known archive") + // ErrEncryptedHeaders: the headers are encrypted, so they cannot be read. + ErrEncryptedHeaders = errors.New("archivehdr: encrypted headers") + // ErrCompressedHeader: a 7z end header is encoded (compressed or encrypted). + ErrCompressedHeader = errors.New("archivehdr: encoded 7z header") + // ErrBudget: the read or block budget ran out before the answer was found. + ErrBudget = errors.New("archivehdr: header budget exhausted") + // ErrMalformed: the header chain is damaged or contradicts itself. + ErrMalformed = errors.New("archivehdr: malformed headers") +) + +// Limits bound one scan. +type Limits struct { + // MaxBlocks is how many header blocks are walked. + MaxBlocks int + // MaxReads is how many reads are made. + MaxReads int + // MaxHeader is the largest header (RAR) that is read. + MaxHeader int + // Max7zHeader is the largest 7z end header that is read. + Max7zHeader int +} + +// DefaultLimits are the limits used by the importer: 64 blocks, 160 reads, and +// 64 KiB per header. +func DefaultLimits() Limits { + return Limits{MaxBlocks: 64, MaxReads: 160, MaxHeader: 64 << 10, Max7zHeader: 64 << 10} +} + +func (l Limits) norm() Limits { + d := DefaultLimits() + if l.MaxBlocks <= 0 { + l.MaxBlocks = d.MaxBlocks + } + if l.MaxReads <= 0 { + l.MaxReads = d.MaxReads + } + if l.MaxHeader <= 0 { + l.MaxHeader = d.MaxHeader + } + if l.Max7zHeader <= 0 { + l.Max7zHeader = d.Max7zHeader + } + return l +} + +// Block is a header that is followed by a data area. +type Block struct { + // HeaderOffset is where the header starts, DataOffset where its data area + // starts and DataSize how long that area is in this volume. + HeaderOffset, DataOffset, DataSize int64 + // UnpackedSize is the size of the whole file once unpacked (-1: unknown). + UnpackedSize int64 + // File is a file entry (not a service block such as a comment or recovery + // record); Dir marks a directory. + File, Dir bool + // Stored says the data is not compressed. + Stored bool + // Encrypted says the data area is AES encrypted; IV is the per-file IV the + // header carries (RAR 5 only). + Encrypted bool + IV []byte + // SplitBefore and SplitAfter say the data continues from the previous and + // into the next volume (RAR only). + SplitBefore, SplitAfter bool +} + +// Volume is what a scan found. +type Volume struct { + Format Format + // Size is the number of bytes of the volume. + Size int64 + // Blocks are the data blocks found in order, up to the offset asked for. + Blocks []Block + // Reads is the number of reads made. + Reads int + // Done says the walk reached the end of the archive or got past the offset + // asked for without error: a block that is not in Blocks does not exist. + Done bool +} + +// reader counts and bounds the reads of one scan. +type reader struct { + r io.ReaderAt + size int64 + reads int + maxReads int +} + +// at reads up to n bytes at off (fewer at the end of the volume). +func (d *reader) at(off int64, n int) ([]byte, error) { + if off < 0 || off > d.size || n < 0 { + return nil, fmt.Errorf("%w: read outside the volume", ErrMalformed) + } + n = int(min(int64(n), d.size-off)) + if n == 0 { + return nil, fmt.Errorf("%w: read at the end of the volume", ErrMalformed) + } + if d.reads++; d.reads > d.maxReads { + return nil, ErrBudget + } + buf := make([]byte, n) + got, err := d.r.ReadAt(buf, off) + if got == n { + return buf, nil + } + if err == nil || errors.Is(err, io.EOF) { + return nil, fmt.Errorf("%w: short read at %d", ErrMalformed, off) + } + return nil, &ReadError{Err: err} +} + +// ReadError wraps an error of the underlying reader (a missing article, a +// timeout): it says nothing about the archive. +type ReadError struct{ Err error } + +func (e *ReadError) Error() string { return "archivehdr: read: " + e.Err.Error() } + +// Unwrap returns the reader's error. +func (e *ReadError) Unwrap() error { return e.Err } + +var ( + rar5Sig = []byte{'R', 'a', 'r', '!', 0x1A, 0x07, 0x01, 0x00} + rar4Sig = []byte{'R', 'a', 'r', '!', 0x1A, 0x07, 0x00} + sevenSg = []byte{'7', 'z', 0xBC, 0xAF, 0x27, 0x1C} +) + +// checkCRC is true outside tests: the fuzz target that explores the parsers +// below the checksums turns it off (a mutated header would otherwise always +// stop at its checksum). +var checkCRC = true + +// Scan identifies the volume and returns its data blocks. upTo is the largest +// data offset the caller is interested in: the walk stops at the first block +// whose header starts beyond it (RAR) and returns everything found so far. +func Scan(r io.ReaderAt, size int64, upTo int64, lim Limits) (*Volume, error) { + lim = lim.norm() + if size <= 0 { + return nil, ErrUnknownFormat + } + d := &reader{r: r, size: size, maxReads: lim.MaxReads} + head, err := d.at(0, 8) + if err != nil { + if errors.Is(err, ErrMalformed) { + return nil, ErrUnknownFormat + } + return nil, err + } + v := &Volume{Size: size} + switch { + case len(head) >= 8 && string(head[:8]) == string(rar5Sig): + v.Format = FormatRAR5 + err = scanRAR5(d, v, upTo, lim) + case len(head) >= 7 && string(head[:7]) == string(rar4Sig): + v.Format = FormatRAR4 + err = scanRAR4(d, v, upTo, lim) + case len(head) >= 6 && string(head[:6]) == string(sevenSg): + v.Format = Format7z + err = scan7z(d, v, lim) + default: + return nil, ErrUnknownFormat + } + v.Reads = d.reads + return v, err +} + +// add returns a+b, or false when it overflows or is negative. +func add(a, b int64) (int64, bool) { + if a < 0 || b < 0 || a > math.MaxInt64-b { + return 0, false + } + return a + b, true +} + +// Outcome is the verdict of Check. +type Outcome uint8 + +// Outcomes. +const ( + // Bound: a stored file's data area starts exactly at the extent, has the + // extent's length here, and is positioned in the file as the extent is. + Bound Outcome = iota + // Unverifiable: the volume could not be judged (unknown format, encoded + // header, budget, damaged chain before the extent). + Unverifiable + // Mismatch: the headers positively contradict the extent. + Mismatch +) + +// Result is the verdict for one extent with the reason. +type Result struct { + Outcome Outcome + Detail string +} + +// Extent is what a bundle claims about one slice of a volume. +type Extent struct { + // Offset and Length are the claimed data area in the volume. + Offset, Length int64 + // Index and Count place the extent in its file (0-based, of Count extents). + Index, Count int + // FileSize is the size of the whole file the extents make up. + FileSize int64 +} + +// Check judges an extent against a scan of its volume. A scan that stopped on +// an error is passed as scanErr: the extent is then unverifiable unless the +// blocks found before the error already contradict it. +func Check(v *Volume, scanErr error, e Extent) Result { + if v == nil { + return Result{Unverifiable, errText(scanErr)} + } + var hit *Block + for k := range v.Blocks { + b := &v.Blocks[k] + if b.DataOffset == e.Offset { + hit = b + break + } + if b.DataOffset < e.Offset && e.Offset < b.DataOffset+b.DataSize { + return Result{Mismatch, fmt.Sprintf("offset %d lies inside the data area of another entry (%d..%d)", e.Offset, b.DataOffset, b.DataOffset+b.DataSize)} + } + } + if hit == nil { + // The walk found no entry starting here. It proves a contradiction only + // when it got past the offset without error. + if scanErr == nil && v.Done { + return Result{Mismatch, fmt.Sprintf("no stored file data starts at offset %d", e.Offset)} + } + return Result{Unverifiable, errText(scanErr)} + } + return checkBlock(v, hit, e) +} + +func errText(err error) string { + if err == nil { + return "the walk ended before the extent" + } + return err.Error() +} + +func checkBlock(v *Volume, b *Block, e Extent) Result { + bad := func(format string, a ...any) Result { return Result{Mismatch, fmt.Sprintf(format, a...)} } + switch { + case !b.File || b.Dir: + return bad("the data area at %d belongs to a service block or a directory, not a file", e.Offset) + case !b.Stored: + return bad("the file at %d is compressed, not stored", e.Offset) + case b.Encrypted: + return bad("the file at %d is itself encrypted but the layout carries no key for it", e.Offset) + case b.DataSize != e.Length: + return bad("the header says %d bytes in this volume, the layout %d", b.DataSize, e.Length) + case b.UnpackedSize >= 0 && b.UnpackedSize != e.FileSize: + return bad("the header says the file is %d bytes, the layout %d", b.UnpackedSize, e.FileSize) + } + end, ok := add(b.DataOffset, b.DataSize) + if !ok || end > v.Size { + return bad("the data area ends beyond the volume (%d of %d bytes)", end, v.Size) + } + if v.Format == Format7z { + if e.Count != 1 { + return Result{Unverifiable, "a 7z file is not split across extents"} + } + return Result{Bound, ""} + } + if first := e.Index == 0; first == b.SplitBefore { + return bad("extent %d of %d: continuation flag before = %v", e.Index+1, e.Count, b.SplitBefore) + } + if last := e.Index == e.Count-1; last == b.SplitAfter { + return bad("extent %d of %d: continuation flag after = %v", e.Index+1, e.Count, b.SplitAfter) + } + return Result{Bound, ""} +} + +// Verify scans a volume and checks one extent against it. It is the function +// the importer calls per extent and the fuzz target exercises. +func Verify(r io.ReaderAt, size int64, e Extent, lim Limits) Result { + if e.Offset < 0 || e.Length < 1 || e.Count < 1 || e.Index < 0 || e.Index >= e.Count { + return Result{Mismatch, "invalid extent"} + } + v, err := Scan(r, size, e.Offset, lim) + if v == nil { + return Result{Unverifiable, errText(err)} + } + return Check(v, err, e) +} diff --git a/internal/peernet/archivehdr/archivehdr_test.go b/internal/peernet/archivehdr/archivehdr_test.go new file mode 100644 index 000000000..f65f62cad --- /dev/null +++ b/internal/peernet/archivehdr/archivehdr_test.go @@ -0,0 +1,294 @@ +package archivehdr + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/kipsilabs/altmount/internal/peernet/archivehdr/archivetest" +) + +func verify(data []byte, e Extent) Result { + return Verify(bytes.NewReader(data), int64(len(data)), e, Limits{}) +} + +func ext(off, length int64, idx, count int, size int64) Extent { + return Extent{Offset: off, Length: length, Index: idx, Count: count, FileSize: size} +} + +func payload(n int) []byte { + out := make([]byte, n) + for i := range out { + out[i] = byte(i%250 + 1) + } + return out +} + +type builder struct { + name string + build func(files ...archivetest.File) archivetest.Built +} + +var rarBuilders = []builder{ + {"rar5", archivetest.RAR5}, + {"rar4", archivetest.RAR4}, +} + +func TestStoredSingleVolumeFileIsBound(t *testing.T) { + for _, b := range rarBuilders { + t.Run(b.name, func(t *testing.T) { + p := payload(1000) + a := b.build(archivetest.File{Name: "movie.mkv", Data: p}) + assert.Equal(t, Bound, verify(a.Data, ext(a.DataOffsets[0], 1000, 0, 1, 1000)).Outcome) + }) + } +} + +func TestSecondFileInTheVolumeIsReachedByWalking(t *testing.T) { + for _, b := range rarBuilders { + t.Run(b.name, func(t *testing.T) { + a := b.build(archivetest.File{Name: "a.nfo", Data: payload(300)}, archivetest.File{Name: "movie.mkv", Data: payload(5000)}) + assert.Equal(t, Bound, verify(a.Data, ext(a.DataOffsets[1], 5000, 0, 1, 5000)).Outcome) + assert.Equal(t, Bound, verify(a.Data, ext(a.DataOffsets[0], 300, 0, 1, 300)).Outcome) + }) + } +} + +func TestExtentThatContradictsTheHeaderIsAMismatch(t *testing.T) { + for _, b := range rarBuilders { + a := b.build(archivetest.File{Name: "movie.mkv", Data: payload(1000)}) + off := a.DataOffsets[0] + cases := map[string]Extent{ + "data starts one byte later": ext(off+1, 999, 0, 1, 1000), + "data starts inside the header": ext(off-4, 1000, 0, 1, 1000), + "length is shorter": ext(off, 999, 0, 1, 1000), + "length is longer": ext(off, 1001, 0, 1, 1000), + "the file is declared bigger": ext(off, 1000, 0, 1, 1001), + "extent is not the first of a file": ext(off, 1000, 1, 2, 1000), + "extent is the first of a split": ext(off, 1000, 0, 2, 2000), + } + for name, e := range cases { + t.Run(b.name+"/"+name, func(t *testing.T) { + r := verify(a.Data, e) + assert.Equal(t, Mismatch, r.Outcome, r.Detail) + }) + } + } +} + +func TestEntryKindsThatCannotBeStreamedAreMismatches(t *testing.T) { + for _, b := range rarBuilders { + t.Run(b.name, func(t *testing.T) { + p := payload(800) + comp := b.build(archivetest.File{Name: "m.mkv", Data: p, Compressed: true}) + r := verify(comp.Data, ext(comp.DataOffsets[0], 800, 0, 1, 800)) + assert.Equal(t, Mismatch, r.Outcome, "a compressed entry is not stored: %s", r.Detail) + svc := b.build(archivetest.File{Name: "CMT", Data: p, Service: true}) + r = verify(svc.Data, ext(svc.DataOffsets[0], 800, 0, 1, 800)) + assert.Equal(t, Mismatch, r.Outcome, "a service block is not a file: %s", r.Detail) + enc := b.build(archivetest.File{Name: "m.mkv", Data: p, IV: bytes.Repeat([]byte{7}, 16)}) + r = verify(enc.Data, ext(enc.DataOffsets[0], 800, 0, 1, 800)) + assert.Equal(t, Mismatch, r.Outcome, "an encrypted entry cannot be read without a key: %s", r.Detail) + }) + } +} + +func TestContinuationFlagsMustAgreeWithTheExtentPosition(t *testing.T) { + for _, b := range rarBuilders { + t.Run(b.name, func(t *testing.T) { + first := b.build(archivetest.File{Name: "m.mkv", Data: payload(500), SplitAfter: true, UnpackedSize: 1200}) + mid := b.build(archivetest.File{Name: "m.mkv", Data: payload(500), SplitBefore: true, SplitAfter: true, UnpackedSize: 1200}) + last := b.build(archivetest.File{Name: "m.mkv", Data: payload(200), SplitBefore: true, UnpackedSize: 1200}) + assert.Equal(t, Bound, verify(first.Data, ext(first.DataOffsets[0], 500, 0, 3, 1200)).Outcome) + assert.Equal(t, Bound, verify(mid.Data, ext(mid.DataOffsets[0], 500, 1, 3, 1200)).Outcome) + assert.Equal(t, Bound, verify(last.Data, ext(last.DataOffsets[0], 200, 2, 3, 1200)).Outcome) + assert.Equal(t, Mismatch, verify(mid.Data, ext(mid.DataOffsets[0], 500, 0, 3, 1200)).Outcome, "a middle part claimed as the first") + assert.Equal(t, Mismatch, verify(first.Data, ext(first.DataOffsets[0], 500, 2, 3, 1200)).Outcome, "a first part claimed as the last") + assert.Equal(t, Mismatch, verify(last.Data, ext(last.DataOffsets[0], 200, 1, 3, 1200)).Outcome, "a last part claimed as a middle one") + }) + } +} + +func TestSevenZip(t *testing.T) { + a := archivetest.SevenZip(archivetest.SevenZipOpts{}, archivetest.File{Name: "a.bin", Data: payload(400)}, archivetest.File{Name: "m.mkv", Data: payload(900)}) + assert.Equal(t, Bound, verify(a.Data, ext(a.DataOffsets[1], 900, 0, 1, 900)).Outcome) + assert.Equal(t, Mismatch, verify(a.Data, ext(a.DataOffsets[1], 899, 0, 1, 900)).Outcome) + assert.Equal(t, Mismatch, verify(a.Data, ext(a.DataOffsets[1]+1, 899, 0, 1, 899)).Outcome, "inside a stored stream") + assert.Equal(t, Unverifiable, verify(a.Data, ext(a.DataOffsets[1], 900, 0, 2, 900)).Outcome, "7z volumes are byte splits, not continuations") + + solid := archivetest.SevenZip(archivetest.SevenZipOpts{Solid: true}, archivetest.File{Name: "a", Data: payload(100)}, archivetest.File{Name: "b", Data: payload(250)}, archivetest.File{Name: "c", Data: payload(77)}) + assert.Equal(t, Bound, verify(solid.Data, ext(solid.DataOffsets[1], 250, 0, 1, 250)).Outcome, "a stored substream of a solid folder") + assert.Equal(t, Bound, verify(solid.Data, ext(solid.DataOffsets[2], 77, 0, 1, 77)).Outcome) + + comp := archivetest.SevenZip(archivetest.SevenZipOpts{}, archivetest.File{Name: "m.mkv", Data: payload(900), Compressed: true}) + assert.Equal(t, Mismatch, verify(comp.Data, ext(comp.DataOffsets[0], 900, 0, 1, 900)).Outcome, "a compressed folder is not stored") + + enc := archivetest.SevenZip(archivetest.SevenZipOpts{EncodedHeader: true}, archivetest.File{Name: "m.mkv", Data: payload(900)}) + r := verify(enc.Data, ext(enc.DataOffsets[0], 900, 0, 1, 900)) + assert.Equal(t, Unverifiable, r.Outcome, "an encoded end header cannot be read: %s", r.Detail) +} + +func TestUnreadableVolumesAreUnverifiableNeverMismatches(t *testing.T) { + a := archivetest.RAR5(archivetest.File{Name: "m.mkv", Data: payload(600)}) + off := a.DataOffsets[0] + junk := bytes.Repeat([]byte{0x5A}, 4096) + corrupt := append([]byte(nil), a.Data...) + corrupt[12] ^= 0xFF // a header checksum + for name, data := range map[string][]byte{ + "not an archive": junk, + "too short": a.Data[:5], + "damaged header": corrupt, + "truncated before the data": a.Data[:off-3], + } { + t.Run(name, func(t *testing.T) { + r := verify(data, ext(off, 600, 0, 1, 600)) + assert.Equal(t, Unverifiable, r.Outcome, r.Detail) + }) + } +} + +func TestEncryptedHeadersAreUnverifiable(t *testing.T) { + // RAR 5: an archive encryption header (type 4) right after the signature. + data := []byte{'R', 'a', 'r', '!', 0x1A, 0x07, 0x01, 0x00} + body := []byte{4, 0, 0, 15, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16} + block := append([]byte{byte(len(body))}, body...) + sum := crc32sum(block) + data = append(append(data, sum[:]...), block...) + data = append(data, make([]byte, 100)...) + r := verify(data, ext(60, 40, 0, 1, 40)) + assert.Equal(t, Unverifiable, r.Outcome, r.Detail) + // RAR 4: main header with the password flag. + d4 := archivetest.RAR4(archivetest.File{Name: "m", Data: payload(50)}).Data + d4[10] |= 0x80 // main header flags, low byte at offset 7+3 + r = verify(d4, ext(70, 50, 0, 1, 50)) + assert.Equal(t, Unverifiable, r.Outcome, r.Detail) +} + +// A volume with many entries before the extent exhausts the walk budget: the +// answer is "unverifiable", not a guess, and the reads stay within the limit. +func TestWalkBudgetIsEnforced(t *testing.T) { + for _, b := range rarBuilders { + t.Run(b.name, func(t *testing.T) { + files := make([]archivetest.File, 300) + for i := range files { + files[i] = archivetest.File{Name: "f", Data: []byte{1, 2, 3}} + } + files = append(files, archivetest.File{Name: "m.mkv", Data: payload(100)}) + a := b.build(files...) + cr := &countingReader{data: a.Data} + lim := Limits{MaxBlocks: 10, MaxReads: 25} + r := Verify(cr, int64(len(a.Data)), ext(a.DataOffsets[300], 100, 0, 1, 100), lim) + assert.Equal(t, Unverifiable, r.Outcome, r.Detail) + assert.LessOrEqual(t, cr.reads, 25) + // With a roomy budget the same extent is bound. + assert.Equal(t, Bound, Verify(bytes.NewReader(a.Data), int64(len(a.Data)), ext(a.DataOffsets[300], 100, 0, 1, 100), Limits{MaxBlocks: 400, MaxReads: 900}).Outcome) + }) + } +} + +type countingReader struct { + data []byte + reads int + bytes int +} + +func (c *countingReader) ReadAt(p []byte, off int64) (int, error) { + c.reads++ + c.bytes += len(p) + return bytes.NewReader(c.data).ReadAt(p, off) +} + +func TestReaderErrorsAreNotAboutTheArchive(t *testing.T) { + a := archivetest.RAR5(archivetest.File{Name: "m.mkv", Data: payload(600)}) + boom := errors.New("article missing") + v, err := Scan(failingReader{err: boom}, int64(len(a.Data)), 100, Limits{}) + assert.Nil(t, v) + var re *ReadError + require.ErrorAs(t, err, &re) + assert.ErrorIs(t, err, boom) +} + +type failingReader struct{ err error } + +func (f failingReader) ReadAt([]byte, int64) (int, error) { return 0, f.err } + +// Hostile declared sizes: nothing is allocated from them and no length +// overflows. +func TestHostileSizesAreRejectedWithoutAllocatingThem(t *testing.T) { + sig := []byte{'R', 'a', 'r', '!', 0x1A, 0x07, 0x01, 0x00} + // RAR 5 header claiming a 2 GiB header. + huge := append(append([]byte{}, sig...), 0, 0, 0, 0, 0x80, 0x80, 0x80, 0x80, 0x08) + huge = append(huge, make([]byte, 64)...) + assert.Equal(t, Unverifiable, verify(huge, ext(100, 10, 0, 1, 10)).Outcome) + // A ten-byte vint that overflows 64 bits. + over := append(append([]byte{}, sig...), 0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F) + assert.Equal(t, Unverifiable, verify(over, ext(100, 10, 0, 1, 10)).Outcome) + // RAR 4 file header with a data size close to 2^63 once the large flag is set. + a := archivetest.RAR4(archivetest.File{Name: "m", Data: payload(10)}) + assert.NotPanics(t, func() { verify(a.Data[:len(a.Data)-30], ext(1<<62, 10, 0, 1, 10)) }) +} + +func TestRealArchivesFromTheImporterFixtures(t *testing.T) { + dir := filepath.Join("..", "..", "importer", "testdata") + read := func(rel string) []byte { + data, err := os.ReadFile(filepath.Join(dir, rel)) + if err != nil { + t.Skipf("fixture %s missing: %v", rel, err) + } + return data + } + single := read("rar_single/archive.rar") + v, err := Scan(bytes.NewReader(single), int64(len(single)), 1<<20, Limits{}) + require.NoError(t, err) + require.Equal(t, FormatRAR5, v.Format) + var blk *Block + for k := range v.Blocks { + if v.Blocks[k].File { + blk = &v.Blocks[k] + } + } + require.NotNil(t, blk) + assert.True(t, blk.Stored) + assert.EqualValues(t, 200000, blk.DataSize) + assert.Equal(t, []byte{0, 1, 2, 3}, single[blk.DataOffset:blk.DataOffset+4], "the data area begins with the payload") + assert.Equal(t, Bound, verify(single, ext(blk.DataOffset, 200000, 0, 1, 200000)).Outcome) + assert.Equal(t, Mismatch, verify(single, ext(blk.DataOffset, 199999, 0, 1, 200000)).Outcome) + + p1, p2 := read("rar_multi/archive.part01.rar"), read("rar_multi/archive.part02.rar") + v1, err := Scan(bytes.NewReader(p1), int64(len(p1)), 1<<20, Limits{}) + require.NoError(t, err) + v2, err := Scan(bytes.NewReader(p2), int64(len(p2)), 1<<20, Limits{}) + require.NoError(t, err) + var f1, f2 Block + for _, b := range v1.Blocks { + if b.File { + f1 = b + } + } + for _, b := range v2.Blocks { + if b.File { + f2 = b + } + } + assert.True(t, f1.SplitAfter && !f1.SplitBefore, "first volume: %+v", f1) + assert.True(t, f2.SplitBefore && !f2.SplitAfter, "last volume: %+v", f2) + assert.EqualValues(t, 200000, f1.DataSize+f2.DataSize) + assert.Equal(t, Bound, verify(p1, ext(f1.DataOffset, f1.DataSize, 0, 2, 200000)).Outcome) + assert.Equal(t, Bound, verify(p2, ext(f2.DataOffset, f2.DataSize, 1, 2, 200000)).Outcome) + assert.Equal(t, Mismatch, verify(p2, ext(f2.DataOffset, f2.DataSize, 0, 2, 200000)).Outcome) + + sz := read("7z_single/archive.7z") + v7, err := Scan(bytes.NewReader(sz), int64(len(sz)), 1<<20, Limits{}) + t.Logf("7z fixture: format=%v blocks=%+v err=%v", v7.Format, v7.Blocks, err) + if err == nil { + require.NotEmpty(t, v7.Blocks) + assert.Equal(t, Bound, verify(sz, ext(v7.Blocks[0].DataOffset, v7.Blocks[0].DataSize, 0, 1, v7.Blocks[0].UnpackedSize)).Outcome) + } else { + assert.ErrorIs(t, err, ErrCompressedHeader, "a real 7z whose end header is encoded is unverifiable") + } +} diff --git a/internal/peernet/archivehdr/archivetest/archivetest.go b/internal/peernet/archivehdr/archivetest/archivetest.go new file mode 100644 index 000000000..8cd68440e --- /dev/null +++ b/internal/peernet/archivehdr/archivetest/archivetest.go @@ -0,0 +1,272 @@ +// Package archivetest builds small stored RAR 4, RAR 5 and 7z archives for +// tests. They are byte-exact enough for the header readers (checksums +// included) and carry the file data verbatim, so offsets can be asserted. +package archivetest + +import ( + "bytes" + "encoding/binary" + "hash/crc32" +) + +// File is one entry of a built archive. +type File struct { + Name string + Data []byte + // Compressed marks the entry as compressed (the data is still written + // verbatim; only the header says otherwise). + Compressed bool + // SplitBefore and SplitAfter set the continuation flags (RAR). + SplitBefore, SplitAfter bool + // UnpackedSize overrides the declared unpacked size (0: len(Data)). + UnpackedSize int64 + // IV, when set, marks the entry encrypted with this IV (RAR 5) or just + // encrypted (RAR 4). + IV []byte + // Service writes the entry as a service block (comment, recovery record). + Service bool +} + +func (f File) unpacked() int64 { + if f.UnpackedSize != 0 { + return f.UnpackedSize + } + return int64(len(f.Data)) +} + +// Built is an archive and where each entry's data starts. +type Built struct { + Data []byte + DataOffsets []int64 +} + +func vint(v uint64) []byte { + var out []byte + for v >= 0x80 { + out = append(out, byte(v)|0x80) + v >>= 7 + } + return append(out, byte(v)) +} + +// RAR5 builds a RAR 5 archive: signature, main header, one block per file, end +// of archive. +func RAR5(files ...File) Built { + var buf bytes.Buffer + buf.Write([]byte{'R', 'a', 'r', '!', 0x1A, 0x07, 0x01, 0x00}) + block := func(content []byte) { + size := vint(uint64(len(content))) + sum := crc32.ChecksumIEEE(append(append([]byte{}, size...), content...)) + var crc [4]byte + binary.LittleEndian.PutUint32(crc[:], sum) + buf.Write(crc[:]) + buf.Write(size) + buf.Write(content) + } + block(append(vint(1), append(vint(0), vint(0)...)...)) // main: type 1, flags 0, archive flags 0 + var offsets []int64 + for _, f := range files { + typ, flags := uint64(2), uint64(0x2) + if f.Service { + typ = 3 + } + if f.SplitBefore { + flags |= 0x8 + } + if f.SplitAfter { + flags |= 0x10 + } + var extra []byte + if f.IV != nil { + rec := append(vint(1), vint(0)...) + rec = append(rec, vint(0)...) + rec = append(rec, 15) + rec = append(rec, make([]byte, 16)...) + rec = append(rec, f.IV...) + extra = append(vint(uint64(len(rec))), rec...) + flags |= 0x1 + } + c := vint(typ) + c = append(c, vint(flags)...) + if extra != nil { + c = append(c, vint(uint64(len(extra)))...) + } + c = append(c, vint(uint64(len(f.Data)))...) + c = append(c, vint(0)...) // file flags + c = append(c, vint(uint64(f.unpacked()))...) + c = append(c, vint(0x20)...) // attributes + comp := uint64(0) + if f.Compressed { + comp = 3 << 7 + } + c = append(c, vint(comp)...) + c = append(c, vint(0)...) // host OS + c = append(c, vint(uint64(len(f.Name)))...) + c = append(c, f.Name...) + c = append(c, extra...) + block(c) + offsets = append(offsets, int64(buf.Len())) + buf.Write(f.Data) + } + block(append(vint(5), append(vint(0), vint(0)...)...)) // end of archive + return Built{Data: buf.Bytes(), DataOffsets: offsets} +} + +// RAR4 builds a RAR 4 archive. +func RAR4(files ...File) Built { + var buf bytes.Buffer + buf.Write([]byte{'R', 'a', 'r', '!', 0x1A, 0x07, 0x00}) + header := func(kind byte, flags uint16, body []byte) { + data := make([]byte, 7+len(body)) + data[2] = kind + binary.LittleEndian.PutUint16(data[3:5], flags) + binary.LittleEndian.PutUint16(data[5:7], uint16(len(data))) + copy(data[7:], body) + binary.LittleEndian.PutUint16(data[:2], uint16(crc32.ChecksumIEEE(data[2:]))) + buf.Write(data) + } + header(0x73, 0, make([]byte, 6)) + var offsets []int64 + for _, f := range files { + body := make([]byte, 25) + binary.LittleEndian.PutUint32(body[:4], uint32(len(f.Data))) + binary.LittleEndian.PutUint32(body[4:8], uint32(f.unpacked())) + body[8] = 3 + binary.LittleEndian.PutUint32(body[9:13], crc32.ChecksumIEEE(f.Data)) + body[17], body[18] = 20, 0x30 + if f.Compressed { + body[18] = 0x33 + } + binary.LittleEndian.PutUint16(body[19:21], uint16(len(f.Name))) + binary.LittleEndian.PutUint32(body[21:25], 0o100644) + flags := uint16(0x8000) + if f.SplitBefore { + flags |= 0x1 + } + if f.SplitAfter { + flags |= 0x2 + } + if f.IV != nil { + flags |= 0x4 + } + kind := byte(0x74) + if f.Service { + kind = 0x7A + } + header(kind, flags, append(body, f.Name...)) + offsets = append(offsets, int64(buf.Len())) + buf.Write(f.Data) + } + header(0x7B, 0, nil) + return Built{Data: buf.Bytes(), DataOffsets: offsets} +} + +func z7num(v uint64) []byte { + if v < 0x80 { + return []byte{byte(v)} + } + for n := 1; n < 8; n++ { + if v < 1<<(7*uint(n+1)) { + first := byte(0xFF<<(8-uint(n))) | byte(v>>(8*uint(n))) + out := []byte{first} + for i := 0; i < n; i++ { + out = append(out, byte(v>>(8*uint(i)))) + } + return out + } + } + out := []byte{0xFF} + for i := 0; i < 8; i++ { + out = append(out, byte(v>>(8*uint(i)))) + } + return out +} + +// SevenZipOpts shape a built 7z archive. +type SevenZipOpts struct { + // Solid puts all files in one folder with one stream each. + Solid bool + // EncodedHeader writes the end header as an encoded (0x17) header. + EncodedHeader bool +} + +// SevenZip builds a 7z archive of stored (copy) folders: the packed streams +// first, then the end header. +func SevenZip(o SevenZipOpts, files ...File) Built { + var packed bytes.Buffer + var offsets []int64 + for _, f := range files { + offsets = append(offsets, 32+int64(packed.Len())) + packed.Write(f.Data) + } + var h bytes.Buffer + h.WriteByte(0x01) // Header + if len(files) > 0 { + h.WriteByte(0x04) // MainStreamsInfo + h.WriteByte(0x06) // PackInfo + h.Write(z7num(0)) + nPacks := len(files) + if o.Solid { + nPacks = 1 + } + h.Write(z7num(uint64(nPacks))) + h.WriteByte(0x09) + if o.Solid { + h.Write(z7num(uint64(packed.Len()))) + } else { + for _, f := range files { + h.Write(z7num(uint64(len(f.Data)))) + } + } + h.WriteByte(0x00) + h.WriteByte(0x07) // UnpackInfo + h.WriteByte(0x0B) + nFolders := nPacks + h.Write(z7num(uint64(nFolders))) + h.WriteByte(0x00) + for i := 0; i < nFolders; i++ { + compressed := !o.Solid && files[i].Compressed + if compressed { + h.Write([]byte{0x01, 0x23, 0x03, 0x01, 0x01, 0x05, 0x5D, 0, 0, 0x10, 0}) // one LZMA coder with props + } else { + h.Write([]byte{0x01, 0x01, 0x00}) // one coder, id size 1, Copy + } + } + h.WriteByte(0x0C) + if o.Solid { + h.Write(z7num(uint64(packed.Len()))) + } else { + for _, f := range files { + h.Write(z7num(uint64(len(f.Data)))) + } + } + h.WriteByte(0x00) + if o.Solid { + h.WriteByte(0x08) // SubStreamsInfo + h.WriteByte(0x0D) + h.Write(z7num(uint64(len(files)))) + h.WriteByte(0x09) + for _, f := range files[:len(files)-1] { + h.Write(z7num(uint64(len(f.Data)))) + } + h.WriteByte(0x00) + } + h.WriteByte(0x00) // end of streams info + } + h.WriteByte(0x05) // FilesInfo + h.Write(z7num(uint64(len(files)))) + h.WriteByte(0x00) + h.WriteByte(0x00) + end := h.Bytes() + if o.EncodedHeader { + end = append([]byte{0x17, 0x06, 0x00, 0x01, 0x09, 0x20, 0x00, 0x00}, make([]byte, 24)...) + } + start := make([]byte, 32) + copy(start, []byte{'7', 'z', 0xBC, 0xAF, 0x27, 0x1C, 0, 4}) + binary.LittleEndian.PutUint64(start[12:20], uint64(packed.Len())) + binary.LittleEndian.PutUint64(start[20:28], uint64(len(end))) + binary.LittleEndian.PutUint32(start[28:32], crc32.ChecksumIEEE(end)) + binary.LittleEndian.PutUint32(start[8:12], crc32.ChecksumIEEE(start[12:32])) + out := append(append(start, packed.Bytes()...), end...) + return Built{Data: out, DataOffsets: offsets} +} diff --git a/internal/peernet/archivehdr/crc_test.go b/internal/peernet/archivehdr/crc_test.go new file mode 100644 index 000000000..99d921b76 --- /dev/null +++ b/internal/peernet/archivehdr/crc_test.go @@ -0,0 +1,13 @@ +package archivehdr + +import ( + "encoding/binary" + "hash/crc32" +) + +// crc32sum is the little-endian CRC-32 of b, as the RAR 5 block header stores it. +func crc32sum(b []byte) [4]byte { + var out [4]byte + binary.LittleEndian.PutUint32(out[:], crc32.ChecksumIEEE(b)) + return out +} diff --git a/internal/peernet/archivehdr/fuzz_test.go b/internal/peernet/archivehdr/fuzz_test.go new file mode 100644 index 000000000..60c0cb67f --- /dev/null +++ b/internal/peernet/archivehdr/fuzz_test.go @@ -0,0 +1,75 @@ +package archivehdr + +import ( + "bytes" + "testing" + + "github.com/kipsilabs/altmount/internal/peernet/archivehdr/archivetest" +) + +// fuzzSeeds are small real stored archives of every format and shape, built +// here (there is no pure Go writer for them outside the tests). +func fuzzSeeds() []archivetest.Built { + p := payload(300) + iv := bytes.Repeat([]byte{9}, 16) + return []archivetest.Built{ + archivetest.RAR5(archivetest.File{Name: "m.mkv", Data: p}), + archivetest.RAR5(archivetest.File{Name: "a", Data: p[:40]}, archivetest.File{Name: "m.mkv", Data: p, SplitBefore: true, SplitAfter: true, UnpackedSize: 900}), + archivetest.RAR5(archivetest.File{Name: "m.mkv", Data: p, IV: iv}, archivetest.File{Name: "c", Data: p[:9], Service: true}), + archivetest.RAR5(archivetest.File{Name: "m.mkv", Data: p, Compressed: true}), + archivetest.RAR4(archivetest.File{Name: "m.mkv", Data: p}), + archivetest.RAR4(archivetest.File{Name: "a", Data: p[:40]}, archivetest.File{Name: "m.mkv", Data: p, SplitBefore: true, UnpackedSize: 600}), + archivetest.RAR4(archivetest.File{Name: "m.mkv", Data: p, IV: iv, Compressed: true}), + archivetest.SevenZip(archivetest.SevenZipOpts{}, archivetest.File{Name: "m.mkv", Data: p}, archivetest.File{Name: "b", Data: p[:50]}), + archivetest.SevenZip(archivetest.SevenZipOpts{Solid: true}, archivetest.File{Name: "a", Data: p[:80]}, archivetest.File{Name: "b", Data: p}), + archivetest.SevenZip(archivetest.SevenZipOpts{}, archivetest.File{Name: "m.mkv", Data: p, Compressed: true}), + archivetest.SevenZip(archivetest.SevenZipOpts{EncodedHeader: true}, archivetest.File{Name: "m.mkv", Data: p}), + } +} + +// FuzzVerifyUnchecked is FuzzVerify with the header checksums not enforced, so +// the fuzzer reaches the parsers under them. +func FuzzVerifyUnchecked(f *testing.F) { + checkCRC = false + defer func() { checkCRC = true }() + addSeeds(f) + f.Fuzz(fuzzVerifyBody) +} + +// FuzzVerify feeds arbitrary bytes and arbitrary extent claims to the header +// check. It must never panic, never read more than its budget, never run +// unbounded and never answer Bound for a claim the volume cannot contain. +func FuzzVerify(f *testing.F) { + addSeeds(f) + f.Fuzz(fuzzVerifyBody) +} + +func addSeeds(f *testing.F) { + for _, s := range fuzzSeeds() { + for _, off := range s.DataOffsets { + f.Add(s.Data, off, int64(300), uint8(0), uint8(1), int64(300)) + } + f.Add(s.Data, int64(40), int64(1), uint8(1), uint8(3), int64(1<<40)) + } + f.Add([]byte{}, int64(0), int64(1), uint8(0), uint8(1), int64(1)) + f.Add([]byte("Rar!\x1a\x07\x01\x00"), int64(8), int64(1), uint8(0), uint8(1), int64(1)) +} + +func fuzzVerifyBody(t *testing.T, data []byte, off, length int64, idx, count uint8, fsize int64) { + lim := Limits{MaxBlocks: 24, MaxReads: 48, MaxHeader: 4096, Max7zHeader: 4096} + e := Extent{Offset: off, Length: length, Index: int(idx), Count: int(count), FileSize: fsize} + cr := &countingReader{data: data} + r := Verify(cr, int64(len(data)), e, lim) + if cr.reads > lim.MaxReads { + t.Fatalf("%d reads, budget %d", cr.reads, lim.MaxReads) + } + if cr.bytes > lim.MaxReads*lim.MaxHeader { + t.Fatalf("%d bytes read, budget %d", cr.bytes, lim.MaxReads*lim.MaxHeader) + } + if r.Outcome == Bound && (off < 0 || length < 1 || off > int64(len(data)) || length > int64(len(data))-off) { + t.Fatalf("bound an extent [%d,+%d) outside a %d byte volume", off, length, len(data)) + } + if r2 := Verify(bytes.NewReader(data), int64(len(data)), e, lim); r2 != r { + t.Fatalf("not deterministic: %+v then %+v", r, r2) + } +} diff --git a/internal/peernet/archivehdr/rar.go b/internal/peernet/archivehdr/rar.go new file mode 100644 index 000000000..cd1fae3a1 --- /dev/null +++ b/internal/peernet/archivehdr/rar.go @@ -0,0 +1,323 @@ +package archivehdr + +import ( + "encoding/binary" + "fmt" + "hash/crc32" +) + +// vint decodes a RAR 5 variable length integer (7 bits per byte, little endian, +// high bit = more) of at most ten bytes. +func vint(b []byte) (v uint64, n int, ok bool) { + for i := 0; i < len(b) && i < 10; i++ { + if i == 9 && b[i] > 1 { + return 0, 0, false + } + v |= uint64(b[i]&0x7f) << (7 * uint(i)) + if b[i]&0x80 == 0 { + return v, i + 1, true + } + } + return 0, 0, false +} + +// cursor reads vints and fixed fields from one header, never past its end. +type cursor struct { + b []byte + err bool +} + +func (c *cursor) vint() uint64 { + v, n, ok := vint(c.b) + if !ok { + c.err = true + return 0 + } + c.b = c.b[n:] + return v +} + +func (c *cursor) skip(n uint64) { + if n > uint64(len(c.b)) { + c.err = true + c.b = nil + return + } + c.b = c.b[n:] +} + +func (c *cursor) take(n int) []byte { + if n < 0 || n > len(c.b) { + c.err = true + c.b = nil + return nil + } + out := c.b[:n] + c.b = c.b[n:] + return out +} + +// RAR 5 block header types and flags. +const ( + r5Main = 1 + r5File = 2 + r5Service = 3 + r5Encrypt = 4 + r5End = 5 + + r5FlagExtra = 0x1 + r5FlagData = 0x2 + r5FlagSplitPrev = 0x8 + r5FlagSplitNext = 0x10 + + r5FileDir = 0x1 + r5FileMTime = 0x2 + r5FileCRC = 0x4 + r5FileUnkSize = 0x8 + r5ExtraEncrypt = 0x1 + maxExtraRecs = 32 +) + +func scanRAR5(d *reader, v *Volume, upTo int64, lim Limits) error { + off := int64(len(rar5Sig)) + for blocks := 0; ; blocks++ { + if off >= d.size || off > upTo { + v.Done = true + return nil + } + if blocks >= lim.MaxBlocks { + return ErrBudget + } + hdr, err := d.at(off, min(1024, lim.MaxHeader)) + if err != nil { + return err + } + if len(hdr) < 6 { + return fmt.Errorf("%w: truncated block header", ErrMalformed) + } + sz, n, ok := vint(hdr[4:]) + if !ok || sz == 0 || sz > uint64(lim.MaxHeader) { + return fmt.Errorf("%w: bad header size", ErrMalformed) + } + total := 4 + n + int(sz) + if int64(total) > d.size-off { + return fmt.Errorf("%w: header runs past the volume", ErrMalformed) + } + if total > len(hdr) { + if hdr, err = d.at(off, total); err != nil { + return err + } + } + hdr = hdr[:total] + if checkCRC && binary.LittleEndian.Uint32(hdr[:4]) != crc32.ChecksumIEEE(hdr[4:]) { + return fmt.Errorf("%w: header checksum", ErrMalformed) + } + blk, kind, err := parseRAR5(hdr[4+n:], off, int64(total)) + if err != nil { + return err + } + switch kind { + case r5Encrypt: + return ErrEncryptedHeaders + case r5End: + v.Done = true + return nil + case r5File, r5Service: + v.Blocks = append(v.Blocks, blk) + } + next, ok := add(blk.DataOffset, blk.DataSize) + if !ok { + return fmt.Errorf("%w: data area overflows", ErrMalformed) + } + off = next + } +} + +// parseRAR5 decodes the part of a block header after its size field. +func parseRAR5(body []byte, off, total int64) (Block, uint64, error) { + bad := func(what string) (Block, uint64, error) { + return Block{}, 0, fmt.Errorf("%w: %s", ErrMalformed, what) + } + c := &cursor{b: body} + typ, flags := c.vint(), c.vint() + var extra, dataSize uint64 + if flags&r5FlagExtra != 0 { + extra = c.vint() + } + if flags&r5FlagData != 0 { + dataSize = c.vint() + } + if c.err || extra > uint64(len(c.b)) || dataSize > 1<<62 { + return bad("block header fields") + } + blk := Block{HeaderOffset: off, DataOffset: off + total, DataSize: int64(dataSize), UnpackedSize: -1, + SplitBefore: flags&r5FlagSplitPrev != 0, SplitAfter: flags&r5FlagSplitNext != 0} + if typ != r5File && typ != r5Service { + return blk, typ, nil + } + blk.File = typ == r5File + // File and service blocks: the extra area is the tail of the header. + extraArea := c.b[uint64(len(c.b))-extra:] + c.b = c.b[:uint64(len(c.b))-extra] + fileFlags := c.vint() + unpacked := c.vint() + c.vint() // attributes + if fileFlags&r5FileMTime != 0 { + c.skip(4) + } + if fileFlags&r5FileCRC != 0 { + c.skip(4) + } + comp := c.vint() + if c.err { + return bad("file header fields") + } + blk.Dir = fileFlags&r5FileDir != 0 + blk.Stored = (comp>>7)&7 == 0 + if fileFlags&r5FileUnkSize == 0 && unpacked <= 1<<62 { + blk.UnpackedSize = int64(unpacked) + } + if err := parseRAR5Extra(extraArea, &blk); err != nil { + return Block{}, 0, err + } + return blk, typ, nil +} + +// parseRAR5Extra reads the extra records of a file header for the file +// encryption record (type 1): version, flags, KDF count, 16-byte salt, 16-byte +// IV. +func parseRAR5Extra(area []byte, blk *Block) error { + c := &cursor{b: area} + for rec := 0; len(c.b) > 0; rec++ { + if rec >= maxExtraRecs { + return fmt.Errorf("%w: too many extra records", ErrMalformed) + } + size := c.vint() + if c.err || size > uint64(len(c.b)) { + return fmt.Errorf("%w: extra record size", ErrMalformed) + } + r := &cursor{b: c.take(int(size))} + if r.vint() != r5ExtraEncrypt || r.err { + continue + } + r.vint() // version + r.vint() // flags + r.take(1) + r.take(16) // salt + iv := r.take(16) + if r.err { + return fmt.Errorf("%w: encryption record", ErrMalformed) + } + blk.Encrypted = true + blk.IV = append([]byte(nil), iv...) + } + return nil +} + +// RAR 4 block types and flags. +const ( + r4Main = 0x73 + r4File = 0x74 + r4Service = 0x7A + r4End = 0x7B + + r4LongBlock = 0x8000 + r4MainPass = 0x80 + r4SplitPrev = 0x01 + r4SplitNext = 0x02 + r4Password = 0x04 + r4DirMask = 0xE0 + r4Large = 0x100 + r4StoreBytes = 0x30 +) + +func scanRAR4(d *reader, v *Volume, upTo int64, lim Limits) error { + off := int64(len(rar4Sig)) + for blocks := 0; ; blocks++ { + if off >= d.size || off > upTo { + v.Done = true + return nil + } + if blocks >= lim.MaxBlocks { + return ErrBudget + } + base, err := d.at(off, 11) + if err != nil { + return err + } + if len(base) < 7 { + return fmt.Errorf("%w: truncated block header", ErrMalformed) + } + typ, flags, hsz := base[2], binary.LittleEndian.Uint16(base[3:5]), int64(binary.LittleEndian.Uint16(base[5:7])) + if hsz < 7 || hsz > int64(lim.MaxHeader) || hsz > d.size-off { + return fmt.Errorf("%w: bad header size", ErrMalformed) + } + hdr := base + if int64(len(hdr)) < hsz { + if hdr, err = d.at(off, int(hsz)); err != nil { + return err + } + } + hdr = hdr[:hsz] + if checkCRC && uint16(crc32.ChecksumIEEE(hdr[2:])) != binary.LittleEndian.Uint16(hdr[:2]) { + return fmt.Errorf("%w: header checksum", ErrMalformed) + } + blk := Block{HeaderOffset: off, DataOffset: off + hsz, UnpackedSize: -1} + switch typ { + case r4End: + v.Done = true + return nil + case r4Main: + if flags&r4MainPass != 0 { + return ErrEncryptedHeaders + } + case r4File, r4Service: + if err := parseRAR4File(hdr, flags, &blk, typ == r4File); err != nil { + return err + } + v.Blocks = append(v.Blocks, blk) + default: + if flags&r4LongBlock != 0 { + if hsz < 11 { + return fmt.Errorf("%w: short block", ErrMalformed) + } + blk.DataSize = int64(binary.LittleEndian.Uint32(hdr[7:11])) + } + } + next, ok := add(blk.DataOffset, blk.DataSize) + if !ok { + return fmt.Errorf("%w: data area overflows", ErrMalformed) + } + off = next + } +} + +// parseRAR4File decodes a file (0x74) or service (0x7A) header. +func parseRAR4File(hdr []byte, flags uint16, blk *Block, file bool) error { + if len(hdr) < 32 { + return fmt.Errorf("%w: short file header", ErrMalformed) + } + pack := uint64(binary.LittleEndian.Uint32(hdr[7:11])) + unp := uint64(binary.LittleEndian.Uint32(hdr[11:15])) + if flags&r4Large != 0 { + if len(hdr) < 40 { + return fmt.Errorf("%w: short large file header", ErrMalformed) + } + pack |= uint64(binary.LittleEndian.Uint32(hdr[32:36])) << 32 + unp |= uint64(binary.LittleEndian.Uint32(hdr[36:40])) << 32 + } + if pack > 1<<62 { + return fmt.Errorf("%w: packed size", ErrMalformed) + } + blk.DataSize = int64(pack) + if unp <= 1<<62 { + blk.UnpackedSize = int64(unp) + } + blk.File = file + blk.Dir = flags&r4DirMask == r4DirMask + blk.Stored = hdr[25] == r4StoreBytes + blk.Encrypted = flags&r4Password != 0 + blk.SplitBefore = flags&r4SplitPrev != 0 + blk.SplitAfter = flags&r4SplitNext != 0 + return nil +} diff --git a/internal/peernet/archivehdr/sevenzip.go b/internal/peernet/archivehdr/sevenzip.go new file mode 100644 index 000000000..252c139e6 --- /dev/null +++ b/internal/peernet/archivehdr/sevenzip.go @@ -0,0 +1,452 @@ +package archivehdr + +import ( + "encoding/binary" + "fmt" + "hash/crc32" +) + +// 7z property ids used here. +const ( + z7End = 0x00 + z7Header = 0x01 + z7ArchiveProps = 0x02 + z7AddStreams = 0x03 + z7MainStreams = 0x04 + z7FilesInfo = 0x05 + z7PackInfo = 0x06 + z7UnpackInfo = 0x07 + z7SubStreams = 0x08 + z7Size = 0x09 + z7CRC = 0x0A + z7FolderID = 0x0B + z7CodersUnpack = 0x0C + z7NumUnpack = 0x0D + z7Encoded = 0x17 + + z7StartHeader = 32 + // z7MaxCount bounds every declared count (folders, streams, coders). + z7MaxCount = 1 << 14 +) + +// z7AES is the method id of the 7z AES coder. +var z7AES = []byte{0x06, 0xF1, 0x07, 0x01} + +// z7Buf reads 7z numbers and bytes from the end header, never past its end. +type z7Buf struct { + b []byte + bad bool +} + +func (z *z7Buf) fail() { z.bad = true; z.b = nil } + +func (z *z7Buf) byte() byte { + if len(z.b) == 0 { + z.fail() + return 0 + } + c := z.b[0] + z.b = z.b[1:] + return c +} + +func (z *z7Buf) take(n uint64) []byte { + if n > uint64(len(z.b)) { + z.fail() + return nil + } + out := z.b[:n] + z.b = z.b[n:] + return out +} + +// number reads a 7z variable length number: the leading one-bits of the first +// byte say how many further bytes (little endian) follow. +func (z *z7Buf) number() uint64 { + first := z.byte() + mask := byte(0x80) + var value uint64 + for i := 0; i < 8; i++ { + if first&mask == 0 { + high := uint64(first & (mask - 1)) + return value | high<<(8*uint(i)) + } + value |= uint64(z.byte()) << (8 * uint(i)) + mask >>= 1 + } + return value +} + +// count reads a number that is a declared element count: bounded, and not more +// than the bytes left could describe. +func (z *z7Buf) count() int { + n := z.number() + if n > z7MaxCount || n > uint64(len(z.b)) { + z.fail() + return 0 + } + return int(n) +} + +// digests skips a digest list of n entries and returns how many are defined. +func (z *z7Buf) digests(n int) (defined []bool) { + defined = make([]bool, n) + if z.byte() != 0 { + for i := range defined { + defined[i] = true + } + } else { + bits := z.take(uint64(n+7) / 8) + for i := range defined { + if i/8 < len(bits) { + defined[i] = bits[i/8]&(0x80>>(uint(i)%8)) != 0 + } + } + } + for _, d := range defined { + if d { + z.take(4) + } + } + return defined +} + +type z7Folder struct { + copyOnly bool + encrypted bool + numPacked int + unpack uint64 + hasCRC bool +} + +func scan7z(d *reader, v *Volume, lim Limits) error { + head, err := d.at(0, z7StartHeader) + if err != nil { + return err + } + if len(head) < z7StartHeader || head[6] != 0 { + return fmt.Errorf("%w: 7z start header", ErrMalformed) + } + if checkCRC && binary.LittleEndian.Uint32(head[8:12]) != crc32.ChecksumIEEE(head[12:32]) { + return fmt.Errorf("%w: 7z start header checksum", ErrMalformed) + } + off, size := binary.LittleEndian.Uint64(head[12:20]), binary.LittleEndian.Uint64(head[20:28]) + if size == 0 { + v.Done = true + return nil + } + if size > uint64(lim.Max7zHeader) || off > uint64(d.size) || size > uint64(d.size) { + return fmt.Errorf("%w: 7z end header outside the volume", ErrMalformed) + } + pos, ok := add(z7StartHeader, int64(off)) + if !ok || pos > d.size-int64(size) { + return fmt.Errorf("%w: 7z end header outside the volume", ErrMalformed) + } + body, err := d.at(pos, int(size)) + if err != nil { + return err + } + if checkCRC && binary.LittleEndian.Uint32(head[28:32]) != crc32.ChecksumIEEE(body) { + return fmt.Errorf("%w: 7z end header checksum", ErrMalformed) + } + z := &z7Buf{b: body} + switch z.byte() { + case z7Header: + case z7Encoded: + return ErrCompressedHeader + default: + return fmt.Errorf("%w: 7z end header type", ErrMalformed) + } + blocks, err := parse7zHeader(z) + if err != nil { + return err + } + v.Blocks = blocks + v.Done = true + return nil +} + +// parse7zHeader walks the plain end header up to the main streams info, which +// is all that is needed: where each folder's packed streams are and how its +// unpacked streams are cut. +func parse7zHeader(z *z7Buf) ([]Block, error) { + for { + switch id := z.byte(); { + case z.bad: + return nil, fmt.Errorf("%w: 7z header", ErrMalformed) + case id == z7End || id == z7FilesInfo: + return nil, nil // no streams: no data areas + case id == z7ArchiveProps: + for { + t := z.byte() + if z.bad || t == z7End { + break + } + z.take(z.number()) + } + case id == z7MainStreams: + return parse7zStreams(z) + default: + return nil, fmt.Errorf("%w: unsupported 7z header property %#x", ErrMalformed, id) + } + } +} + +func parse7zStreams(z *z7Buf) ([]Block, error) { + var ( + packPos uint64 + packSizes []uint64 + folders []z7Folder + subs []int // unpack streams per folder + subSizes [][]uint64 + ) + for { + id := z.byte() + if z.bad { + return nil, fmt.Errorf("%w: 7z streams info", ErrMalformed) + } + switch id { + case z7End: + if subs == nil { // no SubStreamsInfo: one stream per folder + subs = make([]int, len(folders)) + subSizes = make([][]uint64, len(folders)) + for i, f := range folders { + subs[i], subSizes[i] = 1, []uint64{f.unpack} + } + } + return layout7z(packPos, packSizes, folders, subs, subSizes) + case z7PackInfo: + packPos = z.number() + n := z.count() + for { + t := z.byte() + if z.bad || t == z7End { + break + } + switch t { + case z7Size: + for i := 0; i < n; i++ { + packSizes = append(packSizes, z.number()) + } + case z7CRC: + z.digests(n) + default: + z.fail() + } + } + case z7UnpackInfo: + var err error + if folders, err = parse7zFolders(z); err != nil { + return nil, err + } + case z7SubStreams: + var err error + if subs, subSizes, err = parse7zSubStreams(z, folders); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("%w: unsupported 7z streams property %#x", ErrMalformed, id) + } + } +} + +func parse7zFolders(z *z7Buf) ([]z7Folder, error) { + bad := func() ([]z7Folder, error) { return nil, fmt.Errorf("%w: 7z folders", ErrMalformed) } + if z.byte() != z7FolderID { + return bad() + } + n := z.count() + if z.byte() != 0 { // external + return bad() + } + folders := make([]z7Folder, 0, n) + outs := make([]int, 0, n) + for i := 0; i < n && !z.bad; i++ { + coders := z.count() + if coders < 1 || coders > 8 { + return bad() + } + var f z7Folder + var totalIn, totalOut int + for c := 0; c < coders && !z.bad; c++ { + flags := z.byte() + if flags&0x80 != 0 { + return bad() + } + id := z.take(uint64(flags & 0x0F)) + in, out := 1, 1 + if flags&0x10 != 0 { + in, out = z.count(), z.count() + } + if flags&0x20 != 0 { + z.take(z.number()) + } + if string(id) == string(z7AES) { + f.encrypted = true + } + f.copyOnly = coders == 1 && len(id) == 1 && id[0] == 0x00 && in == 1 && out == 1 + totalIn += in + totalOut += out + } + pairs := totalOut - 1 + if pairs < 0 || totalIn-pairs < 1 || z.bad { + return bad() + } + for p := 0; p < pairs*2; p++ { + z.number() + } + f.numPacked = totalIn - pairs + if f.numPacked > 1 { + for p := 0; p < f.numPacked; p++ { + z.number() + } + } + folders = append(folders, f) + outs = append(outs, totalOut) + } + if z.byte() != z7CodersUnpack { + return bad() + } + for i := range folders { + var last uint64 + for o := 0; o < outs[i]; o++ { + last = z.number() + } + folders[i].unpack = last // the last out stream is the folder's result for the layouts that matter + } + for { + t := z.byte() + if z.bad { + return bad() + } + switch t { + case z7End: + return folders, nil + case z7CRC: + for i, def := range z.digests(len(folders)) { + folders[i].hasCRC = def + } + default: + return bad() + } + } +} + +func parse7zSubStreams(z *z7Buf, folders []z7Folder) ([]int, [][]uint64, error) { + bad := func() ([]int, [][]uint64, error) { return nil, nil, fmt.Errorf("%w: 7z substreams", ErrMalformed) } + subs := make([]int, len(folders)) + for i := range subs { + subs[i] = 1 + } + var sizes [][]uint64 + total := 0 + for { + t := z.byte() + if z.bad { + return bad() + } + switch t { + case z7End: + if sizes == nil { + sizes = make([][]uint64, len(folders)) + for i, f := range folders { + if subs[i] == 1 { + sizes[i] = []uint64{f.unpack} + } + } + } + return subs, sizes, nil + case z7NumUnpack: + total = 0 + for i := range subs { + subs[i] = z.count() + if total += subs[i]; total > z7MaxCount { + return bad() + } + } + case z7Size: + sizes = make([][]uint64, len(folders)) + for i, f := range folders { + if subs[i] == 0 { + continue + } + var sum uint64 + for k := 0; k < subs[i]-1; k++ { + s := z.number() + if s > f.unpack || sum > f.unpack-s { + return bad() + } + sum += s + sizes[i] = append(sizes[i], s) + } + sizes[i] = append(sizes[i], f.unpack-sum) + } + case z7CRC: + unknown := 0 + for i, f := range folders { + if subs[i] != 1 || !f.hasCRC { + unknown += subs[i] + } + } + z.digests(unknown) + default: + return bad() + } + } +} + +// layout7z turns the stream tables into data blocks: for each folder, where its +// packed stream starts in the volume and how its unpacked streams are cut. +func layout7z(packPos uint64, packSizes []uint64, folders []z7Folder, subs []int, subSizes [][]uint64) ([]Block, error) { + bad := func() ([]Block, error) { return nil, fmt.Errorf("%w: 7z layout", ErrMalformed) } + if packPos > 1<<62 { + return bad() + } + pos, ok := add(z7StartHeader, int64(packPos)) + if !ok { + return bad() + } + if len(subs) != len(folders) || len(subSizes) != len(folders) { + return bad() + } + var out []Block + next := 0 + for i, f := range folders { + if next+f.numPacked > len(packSizes) { + return bad() + } + start := pos + var packed uint64 + for k := 0; k < f.numPacked; k++ { + ps := packSizes[next+k] + if ps > 1<<62 { + return bad() + } + packed += ps + if pos, ok = add(pos, int64(ps)); !ok { + return bad() + } + } + next += f.numPacked + stored := f.copyOnly && f.numPacked == 1 && packed == f.unpack && !f.encrypted + if subs[i] == 0 { + continue + } + if !stored { + out = append(out, Block{HeaderOffset: start, DataOffset: start, DataSize: int64(packed), UnpackedSize: int64(min(f.unpack, 1<<62)), + File: true, Encrypted: f.encrypted}) + continue + } + at := start + for _, s := range subSizes[i] { + if s > 1<<62 { + return bad() + } + out = append(out, Block{HeaderOffset: start, DataOffset: at, DataSize: int64(s), UnpackedSize: int64(s), File: true, Stored: true}) + if at, ok = add(at, int64(s)); !ok { + return bad() + } + } + } + return out, nil +} diff --git a/internal/peernet/bundleimport/accept.go b/internal/peernet/bundleimport/accept.go index 5572ed393..3129740f9 100644 --- a/internal/peernet/bundleimport/accept.go +++ b/internal/peernet/bundleimport/accept.go @@ -104,11 +104,6 @@ func (i *Importer) Accept(ctx context.Context, req Request) (*Result, error) { return i.rejected(r, err) } b, loc = subset(b, loc, keep) - if req.Origin.Open { - if err := plainLayoutOnly(b); err != nil { - return i.rejected(ReasonLayoutNotAllowedOpen, err) - } - } paths := make([]string, len(loc.Files)) for n, lf := range loc.Files { @@ -138,7 +133,10 @@ func (i *Importer) Accept(ctx context.Context, req Request) (*Result, error) { } return i.rejected(ReasonProbeInconclusive, err) } - if r, attributable, err := i.verifyYenc(ctx, b, req.ExpectedNZBNames); err != nil { + stage, closeStage := i.stager() + defer closeStage() + rep, r, attributable, err := i.verifyYenc(ctx, b, req.ExpectedNZBNames, stage) + if err != nil { if attributable && i.cfg.Reporter != nil { // The layout contradicts the articles themselves: the publisher // served a bad bundle, which is attributable (fetch domain only). @@ -150,7 +148,7 @@ func (i *Importer) Accept(ctx context.Context, req Request) (*Result, error) { } return i.rejected(r, err) } - if r, err := i.probeMedia(ctx, loc, req.Origin.Open); err != nil { + if r, err := i.probeMedia(ctx, loc, stage, needsStructure(rep, b)); err != nil { if r == ReasonMediaInvalid || r == ReasonStructureInvalid { i.noteFailedBundle(ctx, req.Origin.BundleID, b.ReleaseID.String()) } @@ -179,27 +177,6 @@ func (i *Importer) Accept(ctx context.Context, req Request) (*Result, error) { return res, nil } -// plainLayoutOnly requires every written file to be exactly one inline segment -// table: no outer volumes, extents, AES key or clips. Archive layouts cannot be -// bound to the articles without reading archive headers, so the open network -// does not carry them (private networks do). -func plainLayoutOnly(b *bundle.Bundle) error { - for k := range b.Files { - f := &b.Files[k] - switch { - case len(f.Outer) > 0 || len(f.Extents) > 0: - return fmt.Errorf("%s: read through outer archive volumes", f.Name) - case len(f.AESKey) > 0: - return fmt.Errorf("%s: encrypted", f.Name) - case len(f.Clips) > 0: - return fmt.Errorf("%s: clipped", f.Name) - case len(f.Segments) == 0: - return fmt.Errorf("%s: no inline segments", f.Name) - } - } - return nil -} - // noteFailedBundle puts a layout that contradicted the articles or the media on // the negative cache (when the reporter keeps one). func (i *Importer) noteFailedBundle(ctx context.Context, bundleID, releaseID string) { @@ -322,9 +299,40 @@ func probeFiles(loc *bundle.Local) []ProbeFile { return out } +// stageFunc returns the hidden staging area of one accept, creating it on first +// use. +type stageFunc func() (Staging, Reason, error) + +// stager returns the lazily created staging area of one accept and the function +// that removes it (a no-op when it was never created). +func (i *Importer) stager() (stageFunc, func()) { + var st Staging + get := func() (Staging, Reason, error) { + if st != nil { + return st, "", nil + } + if i.cfg.Stager == nil { + return nil, ReasonStaging, errors.New("no stager") + } + s, err := i.cfg.Stager.NewStaging() + if err != nil { + return nil, ReasonStaging, err + } + st = s + return st, "", nil + } + return get, func() { + if st != nil { + _ = st.Close() + } + } +} + // probeMedia reads the head of the main media file(s) through the real reader -// stack, in a hidden staging area that is removed on every outcome. -func (i *Importer) probeMedia(ctx context.Context, loc *bundle.Local, open bool) (Reason, error) { +// stack, in a hidden staging area that is removed on every outcome. strong +// names the files whose archive extents could not be bound to an archive +// header: their container must have a real structural check. +func (i *Importer) probeMedia(ctx context.Context, loc *bundle.Local, stage stageFunc, strong map[string]string) (Reason, error) { var media []bundle.LocalFile for _, lf := range loc.Files { if fileinfo.IsVerifiableMediaFile(lf.Name) { @@ -334,14 +342,10 @@ func (i *Importer) probeMedia(ctx context.Context, loc *bundle.Local, open bool) if len(media) == 0 { return ReasonNoMediaFile, errors.New("no file the media probe can verify") } - if i.cfg.Stager == nil { - return ReasonStaging, errors.New("no stager") - } - st, err := i.cfg.Stager.NewStaging() + st, r, err := stage() if err != nil { - return ReasonStaging, err + return r, err } - defer func() { _ = st.Close() }() for _, lf := range media { if err := st.WriteFileMetadata(lf.Name, lf.Meta); err != nil { return ReasonStaging, err @@ -356,7 +360,11 @@ func (i *Importer) probeMedia(ctx context.Context, loc *bundle.Local, open bool) default: return ReasonMediaInconclusive, fmt.Errorf("%s: %w", lf.Name, res.Err) } - if err := verifyStructure(ctx, st.Opener(), lf.Name, lf.Meta.FileSize, open); err != nil { + why, unbound := strong[lf.Name] + if err := verifyStructure(ctx, st.Opener(), lf.Name, lf.Meta.FileSize, unbound); err != nil { + if unbound && errors.Is(err, errNotVerifiable) { + err = fmt.Errorf("%w (its archive extents could not be checked: %s)", err, why) + } return structureReason(err), fmt.Errorf("%s: %w", lf.Name, err) } } diff --git a/internal/peernet/bundleimport/accept_test.go b/internal/peernet/bundleimport/accept_test.go index 019dd56a6..25a757d91 100644 --- a/internal/peernet/bundleimport/accept_test.go +++ b/internal/peernet/bundleimport/accept_test.go @@ -557,21 +557,16 @@ func TestAccept_ResultCarriesFileSizesInPathOrder(t *testing.T) { } } -// Open network: only plain layouts are accepted (no outer sources, extents, -// AES key or clips); a private network keeps accepting them. -func TestAccept_OpenNetworkRejectsEncryptedLayout(t *testing.T) { +// Every network accepts the same layouts: a keyed layout from the open network +// is accepted when it passes the checks (here the media structure through the +// key), exactly as from a private one. +func TestAccept_OpenNetworkAcceptsKeyedLayout(t *testing.T) { e := newEnv(t) b := e.mkBundle("openaes", fileSpec{name: "movie.mkv", plain: mkvPlain(512), segSize: 128, key: testKey}) req := e.req(b) req.Origin.Open = true _, err := e.imp.Accept(context.Background(), req) - require.Error(t, err) - assert.Equal(t, ReasonLayoutNotAllowedOpen, ReasonOf(err), "%v", err) - e.assertNothingBehind() - - req.Origin.Open = false - _, err = e.imp.Accept(context.Background(), req) - require.NoError(t, err, "a private network may still carry encrypted layouts") + require.NoError(t, err, "%v", err) } func TestAccept_OpenNetworkAcceptsPlainLayout(t *testing.T) { diff --git a/internal/peernet/bundleimport/cost.go b/internal/peernet/bundleimport/cost.go new file mode 100644 index 000000000..467502f52 --- /dev/null +++ b/internal/peernet/bundleimport/cost.go @@ -0,0 +1,56 @@ +package bundleimport + +import ( + "errors" + "sync" +) + +// errCost means the shared cost budget of one accept is used up. +var errCost = errors.New("accept cost budget exhausted") + +// costBudget is the one budget the accept-time reads of a bundle share: the +// yEnc header samples and the archive header reads are charged to it, so a +// hostile bundle cannot multiply its cost by presenting several kinds of work. +// The unit is the declared size of every article touched (the pool reads whole +// articles). +type costBudget struct { + mu sync.Mutex + limit int64 + used int64 + n int +} + +func newCostBudget(limit int64) *costBudget { return &costBudget{limit: limit} } + +// charge takes n bytes from the budget, or reports errCost. +func (c *costBudget) charge(n int64) error { + c.mu.Lock() + defer c.mu.Unlock() + if n < 0 || c.used+n > c.limit { + return errCost + } + c.used += n + c.n++ + return nil +} + +// left is what remains. +func (c *costBudget) left() int64 { + c.mu.Lock() + defer c.mu.Unlock() + return max(c.limit-c.used, 0) +} + +// articles is how many charges were made. +func (c *costBudget) articles() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.n +} + +// spent is what was charged. +func (c *costBudget) spent() int64 { + c.mu.Lock() + defer c.mu.Unlock() + return c.used +} diff --git a/internal/peernet/bundleimport/extents.go b/internal/peernet/bundleimport/extents.go new file mode 100644 index 000000000..03c6fb87e --- /dev/null +++ b/internal/peernet/bundleimport/extents.go @@ -0,0 +1,401 @@ +package bundleimport + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "sync" + + "github.com/javi11/nntppool/v5" + + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/archivehdr" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/utils" +) + +// Archive-extent spot checks. +// +// A file that is read through outer archive volumes is described by extents: +// (outer volume, InnerOffset, InnerLength). No identifier binds them (the IDs +// hash message-IDs only, yEnc sampling binds the OUTER volumes' article sizes +// and offsets), so a publisher could point an extent at any bytes of a volume. +// The archive's own file header is the one thing that says where a stored +// file's data area starts and how long it is in a volume, so each extent is +// compared with the header that precedes its data, read from the user's +// providers through the staged reader (so an encrypted outer volume is read +// decrypted, with the bundle's key, exactly as it will be served). +// +// What is checked per extent: the data area of a STORED file entry (not a +// service block, not compressed, not itself encrypted) starts exactly at +// InnerOffset, is InnerLength long in this volume, the entry's unpacked size is +// the file's size, and its continuation flags fit the extent's place in the file +// (first, middle, last). A 7z volume is judged by its plain end header. +// +// An extent that cannot be judged (unknown format such as a disc image, +// encoded 7z end header, encrypted archive headers, budget spent, damaged +// header chain) is unverifiable: the file is then accepted only when its media +// structure check passes and its container has a real structural check; a +// generic container is rejected (ReasonExtentUnverifiable). + +const ( + // maxExtentChecks bounds how many extents of a bundle are checked. + maxExtentChecks = 64 + // maxVolumeArticles bounds the distinct articles one volume's header walk may + // touch. + maxVolumeArticles = 24 + // extentReserveArticles is what each checked volume is expected to cost, to + // size the shared budget. + extentReserveArticles = 3 +) + +type extentRef struct{ file, ext int } + +// pickExtents chooses the extents to check: all of them up to maxExtentChecks; +// beyond that the first and last extent of the main media file plus a uniform +// random choice (crypto/rand) of the rest. +func pickExtents(b *bundle.Bundle) []extentRef { + var all []extentRef + for fi := range b.Files { + for ei := range b.Files[fi].Extents { + all = append(all, extentRef{fi, ei}) + } + } + if len(all) <= maxExtentChecks { + return all + } + var picks, rest []extentRef + mainFile := mainMediaFile(b) + for _, r := range all { + last := len(b.Files[r.file].Extents) - 1 + if r.file == mainFile && (r.ext == 0 || r.ext == last) { + picks = append(picks, r) + } else { + rest = append(rest, r) + } + } + shuffle(rest) + return append(picks, rest[:maxExtentChecks-len(picks)]...) +} + +// volumeKey identifies one outer volume by content (table, key, volume size). +type volumeKey struct { + table bundle.TableKey + key string + iv string + size int64 +} + +type volumeWork struct { + key volumeKey + outer *bundle.OuterSource + name string + upTo int64 + refs []extentRef + vol *archivehdr.Volume + err error + // wrap is the verdict on the archive that wraps the volume (the header that + // precedes the table's first byte), when the table starts inside its first + // article. + wrap archivehdr.Result +} + +func keyOf(o *bundle.OuterSource) volumeKey { + return volumeKey{table: bundle.KeyOf(o.Segments), key: string(o.AESKey), iv: string(o.AESIV), size: o.VolumeSize} +} + +// groupVolumes lists the distinct outer volumes of the picked extents, with the +// largest offset asked of each. +func groupVolumes(b *bundle.Bundle, picks []extentRef) []*volumeWork { + var out []*volumeWork + index := map[volumeKey]*volumeWork{} + for _, r := range picks { + f := &b.Files[r.file] + e := f.Extents[r.ext] + o := &f.Outer[e.Outer] + k := keyOf(o) + w := index[k] + if w == nil { + w = &volumeWork{key: k, outer: o, name: fmt.Sprintf("outer-%d.bin", len(out))} + index[k] = w + out = append(out, w) + } + w.upTo = max(w.upTo, e.InnerOffset) + w.refs = append(w.refs, r) + } + return out +} + +// extentReserve is the cost the header walks are expected to have: a few +// articles per checked volume. +func extentReserve(b *bundle.Bundle, picks []extentRef) int64 { + var n int64 + for _, w := range groupVolumes(b, picks) { + first := w.outer.Segments[0].Size + n += extentReserveArticles * first + } + return n +} + +// outerMeta is the staged metadata of one outer volume: the volume as a plain +// (or AES) file of its own, read through the same stack as everything else. +func outerMeta(b *bundle.Bundle, o *bundle.OuterSource) *metapb.FileMetadata { + m := &metapb.FileMetadata{Status: metapb.FileStatus_FILE_STATUS_HEALTHY} + var covered int64 + for _, s := range o.Segments { + m.SegmentData = append(m.SegmentData, &metapb.SegmentData{Id: b.Pool[s.Ref], SegmentSize: s.Size, StartOffset: s.Start, EndOffset: s.End}) + covered += s.End - s.Start + 1 + } + m.FileSize = covered + if len(o.AESKey) > 0 { + m.FileSize = o.VolumeSize + m.Encryption = metapb.Encryption_AES + m.AesKey = append([]byte(nil), o.AESKey...) + m.AesIv = append([]byte(nil), o.AESIV...) + } + return m +} + +// chargedReader charges the shared budget for every distinct article a read +// touches (by the table's own geometry) before it reads, and refuses a volume +// more than maxVolumeArticles articles. +type chargedReader struct { + r io.ReaderAt + starts []int64 // starts[k] = decoded offset where usable article k begins + sizes []int64 + seen map[int]struct{} + bud *costBudget +} + +func newChargedReader(r io.ReaderAt, o *bundle.OuterSource, bud *costBudget) *chargedReader { + c := &chargedReader{r: r, seen: map[int]struct{}{}, bud: bud} + var at int64 + for _, s := range o.Segments { + c.starts = append(c.starts, at) + c.sizes = append(c.sizes, s.Size) + at += s.End - s.Start + 1 + } + c.starts = append(c.starts, at) + return c +} + +func (c *chargedReader) ReadAt(p []byte, off int64) (int, error) { + end := off + int64(len(p)) + for k := range c.sizes { + if c.starts[k+1] <= off || c.starts[k] >= end { + continue + } + if _, ok := c.seen[k]; ok { + continue + } + if len(c.seen) >= maxVolumeArticles { + return 0, fmt.Errorf("%w: volume walk touched %d articles", errCost, maxVolumeArticles) + } + if err := c.bud.charge(c.sizes[k]); err != nil { + return 0, err + } + c.seen[k] = struct{}{} + } + return c.r.ReadAt(p, off) +} + +// extentReport says which files of the bundle have extents that were checked +// and could not be bound. +type extentReport struct { + // unbound maps a file index to why one of its checked extents is + // unverifiable. + unbound map[int]string + bound int + checked int +} + +// verifyExtents runs the header check for the picked extents. A mismatch is +// attributable to the publisher. +func (i *Importer) verifyExtents(ctx context.Context, b *bundle.Bundle, st Staging, picks []extentRef, bud *costBudget) (extentReport, Reason, bool, error) { + rep := extentReport{unbound: map[int]string{}} + if len(picks) == 0 { + return rep, "", false, nil + } + volumes := groupVolumes(b, picks) + for _, w := range volumes { + if err := st.WriteFileMetadata(w.name, outerMeta(b, w.outer)); err != nil { + return rep, ReasonStaging, false, err + } + if w.outer.Segments[0].Start > 0 { + if err := st.WriteFileMetadata(w.name+wrapSuffix, rawFromArticleStart(b, w.outer)); err != nil { + return rep, ReasonStaging, false, err + } + } + } + ctx, cancel := context.WithTimeout(ctx, i.cfg.Policy.YencTimeoutOrDefault()) + defer cancel() + var wg sync.WaitGroup + sem := make(chan struct{}, 4) + for _, w := range volumes { + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + w.vol, w.err = scanVolume(ctx, st, w, bud) + w.wrap = checkWrapper(ctx, st, w, bud) + }() + } + wg.Wait() + for _, w := range volumes { + if w.wrap.Outcome == archivehdr.Mismatch { + return rep, ReasonExtentMismatch, true, fmt.Errorf("%s (outer volume): %s", b.Files[w.refs[0].file].Name, w.wrap.Detail) + } + for _, r := range w.refs { + f := &b.Files[r.file] + e := f.Extents[r.ext] + if w.err != nil { + if reason, hard := readFailure(w.err); hard { + return rep, reason, false, fmt.Errorf("%s (volume %d): %w", f.Name, e.Outer+1, w.err) + } + } + res := archivehdr.Check(w.vol, w.err, archivehdr.Extent{Offset: e.InnerOffset, Length: e.InnerLength, + Index: r.ext, Count: len(f.Extents), FileSize: f.Size}) + rep.checked++ + switch res.Outcome { + case archivehdr.Bound: + rep.bound++ + case archivehdr.Mismatch: + return rep, ReasonExtentMismatch, true, fmt.Errorf("%s: extent %d of %d: %s", f.Name, r.ext+1, len(f.Extents), res.Detail) + default: + if _, ok := rep.unbound[r.file]; !ok { + rep.unbound[r.file] = res.Detail + } + } + } + } + return rep, "", false, nil +} + +// readFailure classifies an error of the underlying read that is not about the +// archive: a missing article, a timeout. Budget errors are not hard (the extent +// is then unverifiable). +func readFailure(err error) (Reason, bool) { + var re *archivehdr.ReadError + switch { + case !errors.As(err, &re), errors.Is(err, errCost): + return "", false + case errors.Is(err, nntppool.ErrArticleNotFound): + return ReasonProbeMissing, true + } + return ReasonProbeInconclusive, true +} + +func scanVolume(ctx context.Context, st Staging, w *volumeWork, bud *costBudget) (*archivehdr.Volume, error) { + vctx := context.WithValue(ctx, utils.MaxPrefetchKey, 1) + vctx = context.WithValue(vctx, utils.SuppressStreamTrackingKey, true) + f, err := st.Opener().OpenFile(vctx, w.name, os.O_RDONLY, 0) + if err != nil { + return nil, &archivehdr.ReadError{Err: err} + } + defer func() { _ = f.Close() }() + size := volumeBytes(w.outer) + return archivehdr.Scan(newChargedReader(f, w.outer, bud), size, w.upTo, archivehdr.DefaultLimits()) +} + +// volumeBytes is the readable size of the staged volume. +func volumeBytes(o *bundle.OuterSource) int64 { + if len(o.AESKey) > 0 { + return o.VolumeSize + } + var n int64 + for _, s := range o.Segments { + n += s.End - s.Start + 1 + } + return n +} + +// needsStructure lists the files whose media must pass a real structural check +// because an extent of theirs could not be bound to an archive header. +func needsStructure(rep extentReport, b *bundle.Bundle) map[string]string { + out := map[string]string{} + for fi, why := range rep.unbound { + out[b.Files[fi].Name] = why + } + return out +} + +// wrapSuffix names the staged view of a volume's raw bytes from the start of +// its first article. +const wrapSuffix = ".raw" + +// rawFromArticleStart is the volume's table read raw (no key) from the start of +// its first article, so the archive header that precedes the table's first byte +// is readable. +func rawFromArticleStart(b *bundle.Bundle, o *bundle.OuterSource) *metapb.FileMetadata { + raw := &bundle.OuterSource{Segments: append([]bundle.Segment(nil), o.Segments...)} + raw.Segments[0].Start = 0 + m := outerMeta(b, raw) + return m +} + +// checkWrapper binds the table's first byte to the archive that wraps it. The +// outer volume is itself a stored file of a wrapping archive (usually an +// encrypted one: the bundle's key and IV are that file's); when the table's +// first article starts that archive, the file header ending exactly at the +// table's first byte must be a stored entry whose packed size is the table's +// size and, for a RAR 5 file with a key, whose header IV equals the bundle's. +// A table that does not start inside an archive's first article cannot be +// judged here (the decrypted view still has to be a valid volume). +func checkWrapper(ctx context.Context, st Staging, w *volumeWork, bud *costBudget) archivehdr.Result { + first := w.outer.Segments[0] + if first.Start == 0 { + return archivehdr.Result{Outcome: archivehdr.Unverifiable, Detail: "the volume is a posted file of its own"} + } + raw := &bundle.OuterSource{Segments: append([]bundle.Segment(nil), w.outer.Segments...)} + raw.Segments[0].Start = 0 + vctx := context.WithValue(ctx, utils.MaxPrefetchKey, 1) + vctx = context.WithValue(vctx, utils.SuppressStreamTrackingKey, true) + f, err := st.Opener().OpenFile(vctx, w.name+wrapSuffix, os.O_RDONLY, 0) + if err != nil { + return archivehdr.Result{Outcome: archivehdr.Unverifiable, Detail: err.Error()} + } + defer func() { _ = f.Close() }() + v, err := archivehdr.Scan(newChargedReader(f, raw, bud), volumeBytes(raw), first.Start, archivehdr.DefaultLimits()) + if v == nil || v.Format == archivehdr.Format7z { + return archivehdr.Result{Outcome: archivehdr.Unverifiable, Detail: errString(err)} + } + for k := range v.Blocks { + blk := &v.Blocks[k] + if blk.DataOffset != first.Start { + continue + } + var covered int64 + for _, s := range w.outer.Segments { + covered += s.End - s.Start + 1 + } + bad := func(format string, a ...any) archivehdr.Result { + return archivehdr.Result{Outcome: archivehdr.Mismatch, Detail: fmt.Sprintf(format, a...)} + } + switch { + case !blk.File || blk.Dir || !blk.Stored: + return bad("the entry before the volume's first byte is not a stored file") + case blk.Encrypted != (len(w.outer.AESKey) > 0): + return bad("the entry before the volume's first byte is encrypted = %v, the layout carries a key = %v", blk.Encrypted, len(w.outer.AESKey) > 0) + case blk.IV != nil && !bytes.Equal(blk.IV, w.outer.AESIV): + return bad("the IV of the archive header differs from the layout's") + case blk.DataSize != covered: + return bad("the entry before the volume's first byte holds %d bytes, the volume's articles %d", blk.DataSize, covered) + } + return archivehdr.Result{Outcome: archivehdr.Bound} + } + if err == nil && v.Done { + return archivehdr.Result{Outcome: archivehdr.Mismatch, Detail: fmt.Sprintf("no stored file of the wrapping archive starts at byte %d of the first article", first.Start)} + } + return archivehdr.Result{Outcome: archivehdr.Unverifiable, Detail: errString(err)} +} + +func errString(err error) string { + if err == nil { + return "no answer" + } + return err.Error() +} diff --git a/internal/peernet/bundleimport/extents_test.go b/internal/peernet/bundleimport/extents_test.go new file mode 100644 index 000000000..c5470aa95 --- /dev/null +++ b/internal/peernet/bundleimport/extents_test.go @@ -0,0 +1,441 @@ +package bundleimport + +import ( + "context" + "fmt" + "sort" + "strings" + "testing" + + "github.com/javi11/nntppool/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/archivehdr/archivetest" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/contentid" + "github.com/kipsilabs/altmount/internal/testsupport/fakepool" +) + +// volSpec is one outer volume of a nested test bundle: the bytes of the archive +// (served plain, or AES-CBC encrypted under key; the bundle carries bundleKey). +type volSpec struct { + archive []byte + segSize int + key []byte + bundleKey []byte + // wrap serves the volume as a stored file inside a wrapping RAR 5 archive + // (the shape of an inner volume inside an outer archive): the table then + // starts inside the first article. wrapIV overrides the IV the wrapper's + // header carries; wrapCompressed marks the wrapper's entry compressed. + wrap bool + wrapIV []byte + wrapCompressed bool +} + +// padTo16 zero-pads an archive to a whole number of AES blocks (what a +// decrypted outer volume looks like). +func padTo16(b []byte) []byte { + if n := len(b) % 16; n != 0 { + b = append(append([]byte{}, b...), make([]byte, 16-n)...) + } + return b +} + +// mkNested serves the volumes' articles and returns a valid nzb-scheme bundle +// with one nested file: name, size, read through the volumes by extents. +func (e *env) mkNested(tag, name string, size int64, vols []volSpec, extents []bundle.Extent) *bundle.Bundle { + e.t.Helper() + b := &bundle.Bundle{Scheme: bundle.SchemeNZB, Title: "Test Release " + tag, PostedUnix: 1_700_000_000} + var outers []bundle.OuterSource + for vi, v := range vols { + payload := v.archive + var key, iv []byte + var volSize int64 + if v.key != nil { + payload = cbc(e.t, v.key, padTo16(v.archive)) + key, iv, volSize = v.key, testIV, int64(len(padTo16(v.archive))) + if v.bundleKey != nil { + key = v.bundleKey + } + } + posted, dataStart, dataEnd := payload, 0, len(payload) + if v.wrap { + f := archivetest.File{Name: "inner.part01.rar", Data: payload, Compressed: v.wrapCompressed} + if key != nil { + f.IV = testIV + if v.wrapIV != nil { + f.IV = v.wrapIV + } + } + w := archivetest.RAR5(f) + posted, dataStart = w.Data, int(w.DataOffsets[0]) + dataEnd = dataStart + len(payload) + } + var segs []bundle.Segment + count := 0 + for i := 0; i*v.segSize < len(posted); i++ { + from, end := i*v.segSize, min((i+1)*v.segSize, len(posted)) + id := fmt.Sprintf("%s-v%d-%03d@test", tag, vi, i) + e.fp.SetBehavior(id, fakepoolBehavior(posted[from:end], from, len(posted))) + count++ + b.Pool = append(b.Pool, id) + if end <= dataStart || from >= dataEnd { + continue + } + segs = append(segs, bundle.Segment{Ref: int64(len(b.Pool) - 1), Size: int64(end - from), Start: int64(max(dataStart-from, 0)), End: int64(min(dataEnd, end) - from - 1)}) + } + b.ReleaseFiles = append(b.ReleaseFiles, count) + outers = append(outers, bundle.OuterSource{Segments: segs, AESKey: key, AESIV: iv, VolumeSize: volSize}) + } + b.Files = []bundle.File{{Name: name, Size: size, Outer: outers, Extents: extents}} + b.TotalSize = size + e.seal(b) + return b +} + +func (e *env) seal(b *bundle.Bundle) { + e.t.Helper() + for i := range b.Files { + b.Files[i].FileID = contentid.ID{byte(i + 1)} + } + b.ReleaseID = contentid.ID{9} + loc, err := bundle.ToLocal(b, e.imp.cfg.Now()) + require.NoError(e.t, err) + for i := range b.Files { + id, err := contentid.FileID(loc.Files[i].Meta) + require.NoError(e.t, err) + b.Files[i].FileID = id + } + sort.Slice(b.Files, func(i, j int) bool { return string(b.Files[i].FileID[:]) < string(b.Files[j].FileID[:]) }) + store := &metapb.NzbStore{} + pos := 0 + for _, n := range b.ReleaseFiles { + fe := &metapb.NzbFileEntry{} + for _, id := range b.Pool[pos : pos+n] { + fe.Segments = append(fe.Segments, &metapb.NzbSeg{Id: id}) + } + pos += n + store.Files = append(store.Files, fe) + } + rid, err := contentid.ReleaseID(store) + require.NoError(e.t, err) + b.ReleaseID = rid + require.NoError(e.t, bundle.Validate(b)) +} + +func ext1(vol int, off, length int64) bundle.Extent { + return bundle.Extent{Outer: vol, InnerOffset: off, InnerLength: length} +} + +func (e *env) openReq(b *bundle.Bundle) Request { + req := e.req(b) + req.Origin.Open = true + req.Origin.BundleID = strings.Repeat("e", 64) + return req +} + +// archiveBuilders lists the stored archive formats the importer produces. +var archiveBuilders = map[string]func(files ...archivetest.File) archivetest.Built{ + "rar5": archivetest.RAR5, + "rar4": archivetest.RAR4, +} + +// A nested bundle from the open network is accepted when the extent matches the +// archive's own file header and the media passes. +func TestExtents_OpenNetworkAcceptsNestedAndKeyedLayouts(t *testing.T) { + mkv := mkvPlain(512) + for name, build := range archiveBuilders { + for _, keyed := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/keyed=%v", name, keyed), func(t *testing.T) { + e := newEnv(t) + a := build(archivetest.File{Name: "movie.mkv", Data: mkv}) + v := volSpec{archive: a.Data, segSize: 128} + if keyed { + v.key = testKey + } + b := e.mkNested("n", "movie.mkv", int64(len(mkv)), []volSpec{v}, []bundle.Extent{ext1(0, a.DataOffsets[0], int64(len(mkv)))}) + _, err := e.imp.Accept(context.Background(), e.openReq(b)) + require.NoError(t, err, "%v", err) + }) + } + } +} + +func TestExtents_MultiVolumeFileIsBoundPartByPart(t *testing.T) { + mkv := mkvPlain(600) + for name, build := range archiveBuilders { + t.Run(name, func(t *testing.T) { + e := newEnv(t) + parts := [][]byte{mkv[:200], mkv[200:400], mkv[400:]} + var vols []volSpec + var extents []bundle.Extent + for k, p := range parts { + a := build(archivetest.File{Name: "movie.mkv", Data: p, SplitBefore: k > 0, SplitAfter: k < 2, UnpackedSize: 600}) + vols = append(vols, volSpec{archive: a.Data, segSize: 128}) + extents = append(extents, ext1(k, a.DataOffsets[0], int64(len(p)))) + } + b := e.mkNested("mv", "movie.mkv", 600, vols, extents) + _, err := e.imp.Accept(context.Background(), e.openReq(b)) + require.NoError(t, err, "%v", err) + }) + } +} + +// Every way an extent can disagree with the archive's header is attributable to +// the publisher: rejected, reported, and the layout goes on the negative cache. +func TestExtents_MismatchIsRejectedAndReported(t *testing.T) { + mkv := mkvPlain(512) + type spec struct { + file archivetest.File + extent func(off, n int64) bundle.Extent + } + cases := map[string]spec{ + "offset shifted into the data": {archivetest.File{Name: "movie.mkv", Data: mkv}, func(off, n int64) bundle.Extent { return ext1(0, off+8, n-8) }}, + "offset before the data": {archivetest.File{Name: "movie.mkv", Data: mkv}, func(off, n int64) bundle.Extent { return ext1(0, off-3, n) }}, + "entry is compressed": {archivetest.File{Name: "movie.mkv", Data: mkv, Compressed: true}, func(off, n int64) bundle.Extent { return ext1(0, off, n) }}, + "entry is a service block": {archivetest.File{Name: "movie.mkv", Data: mkv, Service: true}, func(off, n int64) bundle.Extent { return ext1(0, off, n) }}, + "header says another size": {archivetest.File{Name: "movie.mkv", Data: mkv, UnpackedSize: 9999}, func(off, n int64) bundle.Extent { return ext1(0, off, n) }}, + "header says it continues": {archivetest.File{Name: "movie.mkv", Data: mkv, SplitAfter: true}, func(off, n int64) bundle.Extent { return ext1(0, off, n) }}, + } + for name, build := range archiveBuilders { + for label, c := range cases { + t.Run(name+"/"+label, func(t *testing.T) { + e := newEnv(t) + a := build(c.file) + ex := c.extent(a.DataOffsets[0], int64(len(mkv))) + size := ex.InnerLength + b := e.mkNested("mm", "movie.mkv", size, []volSpec{{archive: a.Data, segSize: 128}}, []bundle.Extent{ex}) + _, err := e.imp.Accept(context.Background(), e.openReq(b)) + require.Error(t, err) + assert.Equal(t, ReasonExtentMismatch, ReasonOf(err), "%v", err) + assert.Len(t, e.rep.got, 1, "the publisher is reported") + assert.Equal(t, []string{strings.Repeat("e", 64) + "/" + b.ReleaseID.String()}, e.rep.failedBundles, "the Bundle ID goes on the negative cache") + e.assertNothingBehind() + }) + } + } +} + +// Files whose extents cannot be bound (no archive header can be read) are +// accepted only on a real media structure; a generic container is refused. +func TestExtents_UnverifiableNeedsAStructuralCheck(t *testing.T) { + mkv := mkvPlain(512) + disc := append(append(make([]byte, 40), mkv...), make([]byte, 20)...) // an unknown wrapper: no signature + enc7 := archivetest.SevenZip(archivetest.SevenZipOpts{EncodedHeader: true}, archivetest.File{Name: "movie.mkv", Data: mkv}) + wrappers := map[string]struct { + vol []byte + off int64 + }{ + "unknown wrapper": {disc, 40}, + "encoded 7z header": {enc7.Data, enc7.DataOffsets[0]}, + "encrypted RAR heads": {encryptedRarHeaders(mkv), 33}, + } + for label, w := range wrappers { + t.Run(label+"/matroska is accepted", func(t *testing.T) { + e := newEnv(t) + b := e.mkNested("u", "movie.mkv", int64(len(mkv)), []volSpec{{archive: w.vol, segSize: 128}}, []bundle.Extent{ext1(0, w.off, int64(len(mkv)))}) + _, err := e.imp.Accept(context.Background(), e.openReq(b)) + require.NoError(t, err, "%v", err) + }) + t.Run(label+"/generic container is refused", func(t *testing.T) { + e := newEnv(t) + avi := aviPlain(512) + vol := append(append(make([]byte, int(w.off)), avi...), make([]byte, 20)...) + b := e.mkNested("ug", "movie.avi", int64(len(avi)), []volSpec{{archive: vol, segSize: 128}}, []bundle.Extent{ext1(0, w.off, int64(len(avi)))}) + _, err := e.imp.Accept(context.Background(), e.openReq(b)) + require.Error(t, err) + assert.Equal(t, ReasonExtentUnverifiable, ReasonOf(err), "%v", err) + assert.Empty(t, e.rep.got, "an unverifiable layout is not a proven lie") + e.assertNothingBehind() + }) + } +} + +// aviPlain is a generic container: a RIFF/AVI head the signature check accepts +// and nothing the structure check can read further. +func aviPlain(size int) []byte { + out := junkPlain(size) + copy(out, []byte("RIFF\x00\x00\x00\x00AVI LIST")) + return out +} + +// encryptedRarHeaders is a RAR 5 volume whose first block is the archive +// encryption header, followed by the media bytes. +func encryptedRarHeaders(mkv []byte) []byte { + out := []byte{'R', 'a', 'r', '!', 0x1A, 0x07, 0x01, 0x00, 0, 0, 0, 0, 20, 4, 0, 0, 15} + out = append(out, make([]byte, 16)...) + return append(out, mkv...) +} + +// A wrong key garbles the volume: no header can be read, and the media check +// through the layout rejects it. +func TestExtents_WrongKeyIsRejectedByTheMediaCheck(t *testing.T) { + e := newEnv(t) + mkv := mkvPlain(512) + a := archivetest.RAR5(archivetest.File{Name: "movie.mkv", Data: mkv}) + wrong := []byte("ffffffffffffffffffffffffffffffff") + b := e.mkNested("wk", "movie.mkv", int64(len(mkv)), []volSpec{{archive: a.Data, segSize: 128, key: testKey, bundleKey: wrong}}, + []bundle.Extent{ext1(0, a.DataOffsets[0], int64(len(mkv)))}) + _, err := e.imp.Accept(context.Background(), e.openReq(b)) + require.Error(t, err) + assert.Contains(t, []Reason{ReasonMediaInvalid, ReasonExtentUnverifiable, ReasonStructureInvalid}, ReasonOf(err), "%v", err) + e.assertNothingBehind() +} + +// A missing article of the checked volume is not about the archive. +func TestExtents_MissingArticleIsNotAMismatch(t *testing.T) { + e := newEnv(t) + mkv := mkvPlain(512) + a := archivetest.RAR5(archivetest.File{Name: "movie.mkv", Data: mkv}) + b := e.mkNested("ma", "movie.mkv", int64(len(mkv)), []volSpec{{archive: a.Data, segSize: 128}}, []bundle.Extent{ext1(0, a.DataOffsets[0], int64(len(mkv)))}) + e.fp.SetBehavior(b.Pool[0], fakepoolMissing()) + _, err := e.imp.Accept(context.Background(), e.openReq(b)) + require.Error(t, err) + assert.NotEqual(t, ReasonExtentMismatch, ReasonOf(err), "%v", err) + assert.Empty(t, e.rep.got) +} + +func TestPickExtents_CapsAndAlwaysKeepsTheEndsOfTheMainFile(t *testing.T) { + mk := func(n int) *bundle.Bundle { + ex := make([]bundle.Extent, n) + return &bundle.Bundle{Files: []bundle.File{{Name: "side.nfo", Size: 10, Extents: ex[:5]}, {Name: "movie.mkv", Size: 1 << 30, Extents: ex}}} + } + b := mk(10) + assert.Len(t, pickExtents(b), 15, "every extent is checked up to the cap") + b = mk(300) + seen := map[extentRef]int{} + for range 40 { + picks := pickExtents(b) + require.Len(t, picks, maxExtentChecks) + have := map[extentRef]bool{} + for _, p := range picks { + assert.False(t, have[p], "no extent twice") + have[p] = true + seen[p]++ + } + assert.True(t, have[extentRef{1, 0}] && have[extentRef{1, 299}], "first and last extent of the main media file") + } + assert.Greater(t, len(seen), 150, "the rest is chosen at random") +} + +func TestChargedReader_BoundsTheArticlesOfOneVolumeAndChargesTheBudget(t *testing.T) { + o := &bundle.OuterSource{} + for i := 0; i < 100; i++ { + o.Segments = append(o.Segments, bundle.Segment{Ref: int64(i), Size: 1000, Start: 0, End: 999}) + } + data := make([]byte, 100_000) + bud := newCostBudget(1 << 30) + cr := newChargedReader(bytesReaderAt(data), o, bud) + buf := make([]byte, 10) + for k := 0; k < maxVolumeArticles; k++ { + _, err := cr.ReadAt(buf, int64(k*1000)) + require.NoError(t, err) + _, err = cr.ReadAt(buf, int64(k*1000+500)) // the same article again is free + require.NoError(t, err) + } + assert.EqualValues(t, maxVolumeArticles*1000, bud.spent()) + _, err := cr.ReadAt(buf, 90_000) + require.ErrorIs(t, err, errCost, "one volume may not walk more than %d articles", maxVolumeArticles) + + small := newCostBudget(1500) + cr = newChargedReader(bytesReaderAt(data), o, small) + _, err = cr.ReadAt(buf, 0) + require.NoError(t, err) + _, err = cr.ReadAt(buf, 5000) + require.ErrorIs(t, err, errCost, "the shared budget stops the read") +} + +func fakepoolBehavior(payload []byte, begin, total int) fakepool.SegmentBehavior { + return fakepool.SegmentBehavior{Bytes: payload, YEnc: nntppool.YEncMeta{PartBegin: int64(begin), PartSize: int64(len(payload)), FileSize: int64(total)}} +} + +func fakepoolMissing() fakepool.SegmentBehavior { + return fakepool.SegmentBehavior{Err: nntppool.ErrArticleNotFound} +} + +type bytesReaderAt []byte + +func (b bytesReaderAt) ReadAt(p []byte, off int64) (int, error) { + if off >= int64(len(b)) { + return 0, fmt.Errorf("eof") + } + return copy(p, b[off:]), nil +} + +// The archive that wraps an outer volume is bound too: the header ending at the +// table's first byte must be a stored entry of the table's size whose IV (RAR 5) +// is the layout's. +func TestExtents_WrapperHeaderBindsTheOuterVolume(t *testing.T) { + mkv := mkvPlain(512) + inner := archivetest.RAR5(archivetest.File{Name: "movie.mkv", Data: mkv}) + extents := []bundle.Extent{ext1(0, inner.DataOffsets[0], int64(len(mkv)))} + for _, keyed := range []bool{false, true} { + t.Run(fmt.Sprintf("honest keyed=%v", keyed), func(t *testing.T) { + e := newEnv(t) + v := volSpec{archive: inner.Data, segSize: 256, wrap: true} + if keyed { + v.key = testKey + } + b := e.mkNested("w", "movie.mkv", int64(len(mkv)), []volSpec{v}, extents) + require.Greater(t, b.Files[0].Outer[0].Segments[0].Start, int64(0), "the table starts inside its first article") + _, err := e.imp.Accept(context.Background(), e.openReq(b)) + require.NoError(t, err, "%v", err) + }) + } + bad := map[string]volSpec{ + "header IV is not the layout's": {archive: inner.Data, segSize: 256, wrap: true, key: testKey, wrapIV: []byte("0000000000000000")}, + "wrapping entry is compressed": {archive: inner.Data, segSize: 256, wrap: true, wrapCompressed: true}, + } + for label, v := range bad { + t.Run(label, func(t *testing.T) { + e := newEnv(t) + b := e.mkNested("wb", "movie.mkv", int64(len(mkv)), []volSpec{v}, extents) + _, err := e.imp.Accept(context.Background(), e.openReq(b)) + require.Error(t, err) + assert.Equal(t, ReasonExtentMismatch, ReasonOf(err), "%v", err) + assert.Len(t, e.rep.got, 1) + e.assertNothingBehind() + }) + } +} + +// The cost of the archive header checks for nested releases with many volumes +// (one extent per volume), measured as articles fetched from the pool and the +// declared bytes charged at 768,000 bytes per article. Logged for the +// documentation; the bounds are asserted. +func TestExtents_CostForManyVolumes(t *testing.T) { + const part = 96 + for _, n := range []int{50, 200} { + for _, wrapped := range []bool{false, true} { + t.Run(fmt.Sprintf("%d volumes wrapped=%v", n, wrapped), func(t *testing.T) { + e := newEnv(t) + var vols []volSpec + var extents []bundle.Extent + for k := 0; k < n; k++ { + a := archivetest.RAR5(archivetest.File{Name: "movie.mkv", Data: junkPlain(part), SplitBefore: k > 0, SplitAfter: k < n-1, UnpackedSize: int64(part * n)}) + vols = append(vols, volSpec{archive: a.Data, segSize: 4096, key: testKey, wrap: wrapped}) + extents = append(extents, ext1(k, a.DataOffsets[0], part)) + } + b := e.mkNested(fmt.Sprintf("c%d", n), "movie.mkv", int64(part*n), vols, extents) + e.fp.ResetCounters() + stage, closeStage := e.imp.stager() + defer closeStage() + st, _, err := stage() + require.NoError(t, err) + picks := pickExtents(b) + bud := newCostBudget(1 << 40) + rep, reason, _, err := e.imp.verifyExtents(context.Background(), b, st, picks, bud) + require.NoError(t, err, "%s", reason) + calls := e.fp.TotalCalls() + perArticle := int64(768_000) + t.Logf("%d volumes, wrapped=%v: %d extents checked, %d bound, %d articles fetched, %d article charges (%.0f MB at 768 KB)", + n, wrapped, rep.checked, rep.bound, calls, bud.articles(), float64(int64(bud.articles())*perArticle)/1e6) + assert.Equal(t, min(n, maxExtentChecks), rep.checked) + assert.Equal(t, rep.checked, rep.bound) + assert.LessOrEqual(t, calls, int64(2*maxExtentChecks), "at most two articles per checked volume") + }) + } + } +} diff --git a/internal/peernet/bundleimport/sample.go b/internal/peernet/bundleimport/sample.go index 1684fd6b2..6640e961d 100644 --- a/internal/peernet/bundleimport/sample.go +++ b/internal/peernet/bundleimport/sample.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "slices" "strings" "sync" @@ -68,7 +69,12 @@ type sampleTable struct { nzbName string } -type sample struct{ table, pos int } +// sample is one article to read. A mandatory one must be answered for the +// bundle to be accepted; an optional one that ran out of time is dropped. +type sample struct { + table, pos int + must bool +} // entryNames maps each pool position of a nzb-scheme bundle to the subject // file name of its NZB entry ("" when there is none). @@ -110,16 +116,29 @@ func tablesOf(b *bundle.Bundle, names []string) []sampleTable { } out = append(out, t) } + mainFile := mainMediaFile(b) for i := range b.Files { f := &b.Files[i] - add(f.Name, f.Segments, fileinfo.IsVerifiableMediaFile(f.Name)) + add(f.Name, f.Segments, i == mainFile) for j := range f.Outer { - add(fmt.Sprintf("%s (volume %d)", f.Name, j+1), f.Outer[j].Segments, false) + add(fmt.Sprintf("%s (volume %d)", f.Name, j+1), f.Outer[j].Segments, i == mainFile) } } return out } +// mainMediaFile is the index of the largest media file of the bundle (the one +// whose tables are always sampled in full), or -1. +func mainMediaFile(b *bundle.Bundle) int { + best := -1 + for i := range b.Files { + if fileinfo.IsVerifiableMediaFile(b.Files[i].Name) && (best < 0 || b.Files[i].Size > b.Files[best].Size) { + best = i + } + } + return best +} + // randN returns a uniform value in [0, n) from the operating system's random // source, so a publisher cannot predict which articles will be read. func randN(n int) int { @@ -146,76 +165,127 @@ func shuffle[T any](s []T) { } } -// errTooManyTables means the minimum sampling rate does not fit the budget. +// errTooManyTables means the mandatory samples do not fit the absolute ceiling. var errTooManyTables = errors.New("more segment tables than the sampling budget covers") -// interiorFor returns how many random interior articles a table gets at least -// (one) and in total: tables of media files get one per 500 segments, at least -// three. -func interiorFor(t sampleTable) (minimum, total int) { - n := len(t.ids) - 2 - if n <= 0 { - return 0, 0 - } - total = 1 - if t.main { - total = max(3, len(t.ids)/500) +const ( + // yencFloor is the smallest byte cap of the accept-time sampling. + yencFloor int64 = 64 << 20 + // yencFractionDiv makes the cap one percent of the release size. + yencFractionDiv int64 = 100 + // mediaPerSegments: a table of the main media file gets one interior article + // per this many segments, at least three. + mediaPerSegments = 500 +) + +// yencCap is the byte cap of the sampling for a release of the given declared +// size: one percent of it, at least yencFloor and at most the ceiling. +func yencCap(release, ceiling int64) int64 { + return min(max(release/yencFractionDiv, yencFloor), ceiling) +} + +// releaseBytes is the declared size of all tables. +func releaseBytes(tables []sampleTable) int64 { + var n int64 + for i := range tables { + n += tables[i].total } - return 1, min(total, n) + return n } -// planSamples picks, per table, the first and last segment and at least one -// uniformly random interior one (more for the tables of media files), within a -// cap on the number of articles and on the bytes read. A budget that cannot -// cover every table at the minimum rate is an error, never a silent skip. -func planSamples(tables []sampleTable, limit int, maxBytes int64) ([]sample, error) { - var plan []sample - var bytes int64 +// planMust picks the samples that are never skipped: the LAST article of every +// table (a size lie anywhere in a table shifts every later offset, so the last +// article's part offset and the table total expose it), and for the tables of +// the main media file the first article and random interior ones too (one per +// 500 segments, at least three). +func planMust(tables []sampleTable) (plan []sample, bytes int64) { taken := map[sample]struct{}{} push := func(t, pos int) bool { - s := sample{t, pos} - if _, dup := taken[s]; dup { + s := sample{table: t, pos: pos, must: true} + if _, dup := taken[sample{table: t, pos: pos}]; dup { return false } - taken[s] = struct{}{} + taken[sample{table: t, pos: pos}] = struct{}{} plan = append(plan, s) bytes += tables[t].sizes[pos] return true } - pick := func(t, count int) { - n := len(tables[t].ids) - 2 - for k, tries := 0, 0; k < count && tries < 32*count+32; tries++ { - if push(t, 1+randN(n)) { - k++ - } - } - } - extra := make([]int, len(tables)) for t := range tables { n := len(tables[t].ids) + if n == 0 { + continue + } + push(t, n-1) + if !tables[t].main { + continue + } push(t, 0) - if n > 1 { - push(t, n-1) + want := min(max(3, n/mediaPerSegments), max(n-2, 0)) + for k, tries := 0, 0; k < want && tries < 32*want+32; tries++ { + if push(t, 1+randN(n-2)) { + k++ + } } - lo, hi := interiorFor(tables[t]) - pick(t, lo) - extra[t] = hi - lo } - if len(plan) > limit || bytes > maxBytes { - return nil, fmt.Errorf("%w: %d tables need %d articles (%d bytes), budget %d articles and %d bytes", - errTooManyTables, len(tables), len(plan), bytes, limit, maxBytes) + return plan, bytes +} + +// planOptional spends what is left of the budget on the first article of +// randomly chosen tables and then on one random interior article of randomly +// chosen tables. taken lists the samples already planned. +func planOptional(tables []sampleTable, taken []sample, maxArticles int, maxBytes int64) []sample { + have := make(map[sample]struct{}, len(taken)) + for _, s := range taken { + have[sample{table: s.table, pos: s.pos}] = struct{}{} } + var plan []sample + var bytes int64 order := make([]int, len(tables)) for i := range order { order[i] = i } + try := func(t, pos int) bool { + key := sample{table: t, pos: pos} + if _, dup := have[key]; dup { + return true + } + if len(plan) >= maxArticles || bytes+tables[t].sizes[pos] > maxBytes { + return false + } + have[key] = struct{}{} + plan = append(plan, key) + bytes += tables[t].sizes[pos] + return true + } + order = slices.DeleteFunc(order, func(t int) bool { return tables[t].main }) // already read in full + shuffle(order) + for _, t := range order { + if len(tables[t].ids) > 1 && !try(t, 0) { + return plan + } + } shuffle(order) for _, t := range order { - for k := 0; k < extra[t] && len(plan) < limit && bytes+tables[t].sizes[0] <= maxBytes; k++ { - pick(t, 1) + if n := len(tables[t].ids); n > 2 && !try(t, 1+randN(n-2)) { + return plan } } - return plan, nil + return plan +} + +// planSamples is the whole plan: the mandatory samples, then as much of the +// optional ones as the byte cap (one percent of the release, between 64 MiB +// and the ceiling, raised to fit the mandatory ones) and the article limit +// allow. It fails only when the mandatory samples alone exceed the ceiling or +// the article limit. +func planSamples(tables []sampleTable, limit int, ceiling int64) ([]sample, error) { + must, mustBytes := planMust(tables) + if len(must) > limit || mustBytes > ceiling { + return nil, fmt.Errorf("%w: %d tables need %d articles (%d bytes), ceiling %d articles and %d bytes", + errTooManyTables, len(tables), len(must), mustBytes, limit, ceiling) + } + capBytes := max(yencCap(releaseBytes(tables), ceiling), mustBytes) + return append(must, planOptional(tables, must, limit-len(must), capBytes-mustBytes)...), nil } // SubjectFilename returns the quoted file name of an NZB subject, or "". @@ -239,23 +309,58 @@ func SubjectFilename(subject string) string { // NZB entry the table reads. This is sampling, not proof: the bytes between // sampled articles are unchecked. attributable says the publisher's layout // contradicts the articles (not merely the local NZB, or a missing article). -func (i *Importer) verifyYenc(ctx context.Context, b *bundle.Bundle, names []string) (reason Reason, attributable bool, err error) { +// +// It is also the one place the accept-time reads share a budget: the mandatory +// samples (the last article of every table, more for the main media file), then +// the archive-extent header checks, then as many optional samples (first and +// interior articles of other tables, at random) as the remaining budget allows. +// The returned report says which files have extents that could not be bound to +// an archive header. +func (i *Importer) verifyYenc(ctx context.Context, b *bundle.Bundle, names []string, stage stageFunc) (rep extentReport, reason Reason, attributable bool, err error) { if i.cfg.Headers == nil { - return ReasonProbeInconclusive, false, errors.New("no way to read article headers") + return rep, ReasonProbeInconclusive, false, errors.New("no way to read article headers") } tables := tablesOf(b, names) - plan, perr := planSamples(tables, i.cfg.Policy.YencSamples(), i.cfg.Policy.YencBytes()) - if perr != nil { - return ReasonTooManyTables, false, perr + picks := pickExtents(b) + reserve := extentReserve(b, picks) + must, mustBytes := planMust(tables) + limit, ceiling := i.cfg.Policy.YencSamples(), i.cfg.Policy.YencBytes() + if len(must) > limit || mustBytes+reserve > ceiling { + return rep, ReasonTooManyTables, false, fmt.Errorf("%w: %d tables need %d articles (%d bytes plus %d for archive headers), ceiling %d articles and %d bytes", + errTooManyTables, len(tables), len(must), mustBytes, reserve, limit, ceiling) } + bud := newCostBudget(max(yencCap(releaseBytes(tables), ceiling), mustBytes+reserve)) + _ = bud.charge(mustBytes) ctx, cancel := context.WithTimeout(ctx, i.cfg.Policy.YencTimeoutOrDefault()) defer cancel() - results := i.runSamples(ctx, tables, plan) + if reason, attributable, err := judge(i.runSamples(ctx, tables, must)); err != nil { + return rep, reason, attributable, err + } + if len(picks) > 0 { + st, r, serr := stage() + if serr != nil { + return rep, r, false, serr + } + if rep, reason, attributable, err = i.verifyExtents(ctx, b, st, picks, bud); err != nil { + return rep, reason, attributable, err + } + } + optional := planOptional(tables, must, limit-len(must), bud.left()) + for _, s := range optional { + _ = bud.charge(tables[s.table].sizes[s.pos]) + } + reason, attributable, err = judge(i.runSamples(ctx, tables, optional)) + return rep, reason, attributable, err +} + +// judge turns the verdicts of a run into one: a contradiction outranks a +// missing article. +func judge(results []sampleVerdict) (Reason, bool, error) { var first *sampleVerdict for k := range results { r := &results[k] if r.reason == ReasonYencMismatch { - return r.reason, r.attributable, r.err // a contradiction outranks a missing article + return r.reason, r.attributable, r.err } if r.reason != "" && first == nil { first = r @@ -290,6 +395,8 @@ func (i *Importer) runSamples(ctx context.Context, tables []sampleTable, plan [] switch { case errors.Is(err, nntppool.ErrArticleNotFound): results[k] = sampleVerdict{reason: ReasonProbeMissing, err: fmt.Errorf("%s: article %d is missing", t.name, s.pos)} + case err != nil && !s.must && (errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled)): + // An optional sample that ran out of time is dropped, not a failure. case err != nil: results[k] = sampleVerdict{reason: ReasonProbeInconclusive, err: fmt.Errorf("%s: %w", t.name, err)} case h.PartSize != t.sizes[s.pos] || h.PartBegin != t.prefix[s.pos] || h.FileSize != t.total: diff --git a/internal/peernet/bundleimport/structure.go b/internal/peernet/bundleimport/structure.go index 4bda6b888..02b92fb7d 100644 --- a/internal/peernet/bundleimport/structure.go +++ b/internal/peernet/bundleimport/structure.go @@ -37,7 +37,10 @@ import ( // MPEG-TS / M2TS the sync byte on the packet grid (188, 192 with the // 4-byte timestamp, or 204 bytes) at the start, quartiles and // the last packet. -// anything else full-length reads at the quartiles and the last bytes. +// anything else full-length reads at the quartiles and the last bytes. A +// file whose archive extents could not be bound to an archive +// header (strong) is refused instead: readable windows prove +// nothing about bytes the layout may have misplaced. // // There is no deeper media parser in the repository (no MediaStructure probe), // so this is the strongest check available without adding one. @@ -51,8 +54,9 @@ const ( var errStructure = errors.New("media structure is not valid") -// errNotVerifiable says the container has no structural check here: the open -// network does not carry such media. +// errNotVerifiable says a file whose archive extents could not be bound to a +// header has a container with no structural check, so readable windows are not +// enough. var errNotVerifiable = errors.New("container has no structural check") type readerAt struct { @@ -102,7 +106,7 @@ func (r *readerAt) readRaw(off int64, n int) ([]byte, error) { } // verifyStructure runs the container check for the staged file name. -func verifyStructure(ctx context.Context, op contentverify.Opener, name string, size int64, open bool) error { +func verifyStructure(ctx context.Context, op contentverify.Opener, name string, size int64, strong bool) error { ctx, cancel := context.WithTimeout(ctx, structReadTimeout) defer cancel() ctx = context.WithValue(ctx, utils.MaxPrefetchKey, 1) @@ -125,7 +129,7 @@ func verifyStructure(ctx context.Context, op contentverify.Opener, name string, case tsStride(r) != 0: return verifyTS(r, tsStride(r)) } - if open { + if strong { return fmt.Errorf("%w: %s", errNotVerifiable, name) } return verifyWindows(r) @@ -579,7 +583,7 @@ func parseSeek(b []byte) (id []byte, pos int64, err error) { func structureReason(err error) Reason { switch { case errors.Is(err, errNotVerifiable): - return ReasonContainerNotVerifiable + return ReasonExtentUnverifiable case errors.Is(err, errStructure): return ReasonStructureInvalid case errors.Is(err, nntppool.ErrArticleNotFound): diff --git a/internal/peernet/bundleimport/structure_forgery_test.go b/internal/peernet/bundleimport/structure_forgery_test.go index dcafc2094..2c38dee4e 100644 --- a/internal/peernet/bundleimport/structure_forgery_test.go +++ b/internal/peernet/bundleimport/structure_forgery_test.go @@ -10,11 +10,11 @@ import ( "github.com/stretchr/testify/require" ) -func structureOpen(t *testing.T, data []byte, name string, open bool) error { +func structureOpen(t *testing.T, data []byte, name string) error { t.Helper() fs := afero.NewMemMapFs() require.NoError(t, afero.WriteFile(fs, name, data, 0o644)) - return verifyStructure(context.Background(), memOpener{fs}, name, int64(len(data)), open) + return verifyStructure(context.Background(), memOpener{fs}, name, int64(len(data)), false) } func box(typ string, payload []byte) []byte { @@ -34,7 +34,7 @@ func TestForgery_FragmentedMP4WithGarbageMiddleAndFreeTail(t *testing.T) { } b = append(b, bytes.Repeat([]byte{0xAB}, 3<<20)...) b = append(b, box("free", nil)...) - err := structureOpen(t, b, "a.mp4", false) + err := structureOpen(t, b, "a.mp4") require.Error(t, err) assert.ErrorIs(t, err, errStructure) } @@ -57,15 +57,15 @@ func TestForgery_RealFragmentTailsStillPass(t *testing.T) { "moof+mdat then mfra": append(append([]byte{}, head...), mfra()...), "mfra then free, skip": append(append(append([]byte{}, head...), mfra()...), append(mp4Box("free", 10), mp4Box("skip", 12)...)...), } { - t.Run(name, func(t *testing.T) { assert.NoError(t, structureOpen(t, data, "a.mp4", false)) }) + t.Run(name, func(t *testing.T) { assert.NoError(t, structureOpen(t, data, "a.mp4")) }) } bad := append([]byte{}, head...) bad = append(bad, box("mfra", box("mfro", []byte{0, 0, 0, 0, 0, 0, 0, 99}))...) // mfro lies about its mfra - assert.Error(t, structureOpen(t, bad, "a.mp4", false), "an mfro that does not describe its mfra") + assert.Error(t, structureOpen(t, bad, "a.mp4"), "an mfro that does not describe its mfra") lone := append(append([]byte{}, head[:len(head)-len(mp4Box("mdat", 200))]...), mp4Box("mdat", 200)...) _ = lone notPair := append(append([]byte{}, head...), mp4Box("sidx", 40)...) - assert.Error(t, structureOpen(t, notPair, "a.mp4", false), "a tail that is neither a moof+mdat pair nor an mfra") + assert.Error(t, structureOpen(t, notPair, "a.mp4"), "a tail that is neither a moof+mdat pair nor an mfra") } func TestStructure_NonFragmentedMP4WithManyTopLevelBoxes(t *testing.T) { @@ -75,7 +75,7 @@ func TestStructure_NonFragmentedMP4WithManyTopLevelBoxes(t *testing.T) { } data = append(data, mp4Box("moov", 200)...) data = append(data, mp4Box("mdat", 600)...) - assert.NoError(t, structureOpen(t, data, "a.mp4", false), "a legitimate file with more than 63 boxes is not a 'too many reads' rejection") + assert.NoError(t, structureOpen(t, data, "a.mp4"), "a legitimate file with more than 63 boxes is not a 'too many reads' rejection") } // Reviewer forgery: a Matroska with valid head (SeekHead promising Cues at the @@ -102,7 +102,7 @@ func forgedMKV(total int, fill func(b []byte)) []byte { } func TestForgery_MatroskaWithValidHeadAndTailButZeroMiddle(t *testing.T) { - err := structureOpen(t, forgedMKV(4<<20, nil), "a.mkv", false) + err := structureOpen(t, forgedMKV(4<<20, nil), "a.mkv") require.Error(t, err) assert.ErrorIs(t, err, errStructure) } @@ -110,7 +110,7 @@ func TestForgery_MatroskaWithValidHeadAndTailButZeroMiddle(t *testing.T) { // The same shape with real SimpleBlock chains in the middle is a real file. func TestForgery_MatroskaWithRealClustersStillPasses(t *testing.T) { data := forgedMKV(4<<20, func(b []byte) { fillSimpleBlocks(b, 200, 3000) }) - assert.NoError(t, structureOpen(t, data, "a.mkv", false)) + assert.NoError(t, structureOpen(t, data, "a.mkv")) } func TestForgery_TransportStreamWithLostSyncInTheMiddle(t *testing.T) { @@ -124,27 +124,26 @@ func TestForgery_TransportStreamWithLostSyncInTheMiddle(t *testing.T) { } hit := 0 for range 40 { - if structureOpen(t, ts, "a.ts", false) != nil { + if structureOpen(t, ts, "a.ts") != nil { hit++ } } assert.Greater(t, hit, 30, "random grid positions must find the broken band almost every time") } -// The open network only carries containers with a real structural check. -func TestOpenNetwork_GenericContainersAreNotVerifiable(t *testing.T) { +// Every network accepts the same containers: a generic one (avi, mpg, ...) +// gets the head signature and readable-windows check wherever it came from +// (the structure check no longer depends on the network). +func TestGenericContainersAreAcceptedEverywhere(t *testing.T) { generic := make([]byte, 5000) for i := range generic { generic[i] = byte(i%251 + 1) } - err := structureOpen(t, generic, "a.avi", true) - require.Error(t, err) - assert.Equal(t, ReasonContainerNotVerifiable, structureReason(err)) - assert.NoError(t, structureOpen(t, generic, "a.avi", false), "private networks keep the readable-only check") - assert.NoError(t, structureOpen(t, mkvPlain(4096), "a.mkv", true), "Matroska is verified, so open accepts it") - assert.NoError(t, structureOpen(t, tsFile(188, 0, 60), "a.ts", true)) - assert.Equal(t, ReasonContainerNotVerifiable, structureReason(structureOpen(t, append(make([]byte, 77), tsFile(188, 0, 60)...), "a.ts", true)), - "a capture that starts mid-packet is read as an unknown container") + assert.NoError(t, structureOpen(t, generic, "a.avi"), "readable windows are enough for a generic container") + assert.NoError(t, structureOpen(t, mkvPlain(4096), "a.mkv")) + assert.NoError(t, structureOpen(t, tsFile(188, 0, 60), "a.ts")) + assert.NoError(t, structureOpen(t, append(make([]byte, 77), tsFile(188, 0, 60)...), "a.ts"), + "a capture that starts mid-packet is read as a generic container") } // simpleBlocks returns exactly n bytes (n >= 7) of consecutive Matroska diff --git a/internal/peernet/bundleimport/types.go b/internal/peernet/bundleimport/types.go index 02e6a4091..7f6ef51ed 100644 --- a/internal/peernet/bundleimport/types.go +++ b/internal/peernet/bundleimport/types.go @@ -262,11 +262,12 @@ const ( ReasonStructureInvalid Reason = "structure_invalid" // media structure check failed beyond the first bytes ReasonForbiddenFile Reason = "forbidden_file" // a file a media server or the OS could act on ReasonSidecarTooLarge Reason = "sidecar_too_large" // a subtitle or nfo over its size cap - // ReasonLayoutNotAllowedOpen: the open network only carries plain layouts. - ReasonLayoutNotAllowedOpen Reason = "layout_not_allowed_open" - // ReasonContainerNotVerifiable: the open network only carries media whose - // container has a real structural check. - ReasonContainerNotVerifiable Reason = "container_not_verifiable" + // ReasonExtentMismatch: an archive extent contradicts the archive's own file + // header (attributable to the publisher). + ReasonExtentMismatch Reason = "extent_mismatch" + // ReasonExtentUnverifiable: a file's archive extents could not be bound to + // a header and its container has no structural check to stand in. + ReasonExtentUnverifiable Reason = "extent_unverifiable" // ReasonTooManyTables: the layout has more segment tables than the yEnc // sampling budget can cover at the minimum rate. ReasonTooManyTables Reason = "too_many_tables" @@ -310,10 +311,12 @@ type Policy struct { // AcceptTimeout bounds the whole of Accept. AcceptTimeout time.Duration // MaxYencSamples caps the articles whose yEnc headers are read, MaxYencBytes - // the declared bytes those reads cost (the pool reads whole articles) and - // YencTimeout the time of the whole sampling. Every segment table gets its - // first, last and at least one random interior article; a layout whose - // tables do not fit the cap at that rate is rejected, never left unsampled. + // is the absolute ceiling of the declared bytes those reads (and the archive + // header reads) cost (the pool reads whole articles) and YencTimeout the + // time of the whole sampling. Every segment table gets its last article + // read; the byte cap is one percent of the release, at least 64 MiB and at + // most MaxYencBytes (raised to fit the last articles), and a layout whose + // last articles alone exceed the ceiling is rejected, never left unsampled. MaxYencSamples int MaxYencBytes int64 YencTimeout time.Duration @@ -334,8 +337,8 @@ func DefaultPolicy() Policy { MaxMediaProbes: 4, MediaProbeTimeout: 30 * time.Second, AcceptTimeout: 2 * time.Minute, - MaxYencSamples: 600, - MaxYencBytes: 512 << 20, + MaxYencSamples: 2000, + MaxYencBytes: 1 << 30, YencTimeout: 90 * time.Second, MaxMediaFiles: 40, MaxSidecars: 50, @@ -470,6 +473,10 @@ func (p Policy) YencBytes() int64 { return p.MaxYencBytes } +// fullCheckHeaderBytes caps the declared bytes of the forced full check's +// header sample. +const fullCheckHeaderBytes int64 = 512 << 20 + // YencTimeoutOrDefault is the time budget of the yEnc sampling. func (p Policy) YencTimeoutOrDefault() time.Duration { if p.YencTimeout <= 0 { diff --git a/internal/peernet/bundleimport/verify_headers.go b/internal/peernet/bundleimport/verify_headers.go index 771341c91..d21cc6617 100644 --- a/internal/peernet/bundleimport/verify_headers.go +++ b/internal/peernet/bundleimport/verify_headers.go @@ -85,11 +85,11 @@ func (i *Importer) VerifyPeerHeaders(ctx context.Context, filePath string) error g -= len(tables[ti].ids) ti++ } - s := sample{ti, g} + s := sample{table: ti, pos: g, must: true} if _, dup := picked[s]; dup { continue } - if bytes+tables[ti].sizes[g] > i.cfg.Policy.YencBytes() { + if bytes+tables[ti].sizes[g] > fullCheckHeaderBytes { break } picked[s] = struct{}{} diff --git a/internal/peernet/bundleimport/yenc_test.go b/internal/peernet/bundleimport/yenc_test.go index 4e94522b9..0e2e16c2b 100644 --- a/internal/peernet/bundleimport/yenc_test.go +++ b/internal/peernet/bundleimport/yenc_test.go @@ -5,6 +5,7 @@ import ( "fmt" "sync/atomic" "testing" + "time" "github.com/javi11/nntppool/v5" "github.com/stretchr/testify/assert" @@ -116,11 +117,35 @@ func mkTable(n int, main bool) sampleTable { t := sampleTable{ids: make([]string, n), sizes: make([]int64, n), prefix: make([]int64, n), main: main} for i := range t.sizes { t.sizes[i] = 100 + t.prefix[i] = int64(i) * 100 + t.total += 100 } return t } -func TestPlanSamples_EveryTableGetsFirstLastAndAnInterior(t *testing.T) { +// bigTable is a table of n articles of 768,000 bytes. +func bigTable(n int, main bool) sampleTable { + t := mkTable(n, main) + t.total = 0 + for i := range t.sizes { + t.sizes[i] = 768_000 + t.prefix[i] = t.total + t.total += 768_000 + } + return t +} + +func planBytes(tables []sampleTable, plan []sample) (n int, bytes int64) { + for _, s := range plan { + bytes += tables[s.table].sizes[s.pos] + } + return len(plan), bytes +} + +// Every table's LAST article is always read (a size lie anywhere in a table +// shifts the offsets after it, so the last article shows it); the first and a +// random interior one are read when the budget allows. +func TestPlanSamples_EveryTableGetsItsLastArticle(t *testing.T) { tables := make([]sampleTable, 50) for i := range tables { tables[i] = mkTable(10, false) @@ -132,40 +157,85 @@ func TestPlanSamples_EveryTableGetsFirstLastAndAnInterior(t *testing.T) { per[s.table] = append(per[s.table], s.pos) } require.Len(t, per, 50, "no table is left unsampled") - seen := map[int]struct{}{} for _, ps := range per { - require.Len(t, ps, 3) - assert.Contains(t, ps, 0) - assert.Contains(t, ps, 9) - for _, p := range ps { - if p != 0 && p != 9 { - seen[p] = struct{}{} - } - } + assert.Contains(t, ps, 9, "the last article of every table") + assert.Contains(t, ps, 0, "a roomy budget also reads the first") + assert.Len(t, ps, 3, "and one interior article") } - assert.Greater(t, len(seen), 3, "interior picks vary") } func TestPlanSamples_MediaTablesAreSampledDenserAndTheCapHolds(t *testing.T) { plan, err := planSamples([]sampleTable{mkTable(5000, true)}, 600, 1<<30) require.NoError(t, err) assert.Len(t, plan, 2+10, "first, last and one interior per 500 segments") - plan, err = planSamples([]sampleTable{mkTable(5000, true), mkTable(10, false)}, 8, 1<<30) + // Scarce article budget: the main file's tables are still read in full at + // that rate; the rest of the release gets only what is left. + plan, err = planSamples([]sampleTable{mkTable(5000, true), mkTable(10, false), mkTable(10, false)}, 14, 1<<30) require.NoError(t, err) - assert.LessOrEqual(t, len(plan), 8) + assert.LessOrEqual(t, len(plan), 14) + main := 0 + for _, s := range plan { + if s.table == 0 { + main++ + } + } + assert.Equal(t, 12, main) } -func TestPlanSamples_RejectsWhenTheMinimumDoesNotFit(t *testing.T) { - tables := make([]sampleTable, 20) +// A release with many small volumes is not rejected: the last article of each +// is read, and the byte cap is raised to fit them (it only ever refuses beyond +// the absolute ceiling). +func TestPlanSamples_ManyVolumesAreThinnedNotRejected(t *testing.T) { + tables := make([]sampleTable, 300) for i := range tables { - tables[i] = mkTable(10, false) + tables[i] = bigTable(20, false) // 15 MB volumes: a 4.6 GB release + } + plan, err := planSamples(tables, 2000, 1<<30) + require.NoError(t, err, "300 volumes must not be rejected") + n, bytes := planBytes(tables, plan) + assert.Equal(t, 300, n, "the last article of every volume, and nothing else fits") + assert.EqualValues(t, 300*768_000, bytes) + for _, s := range plan { + assert.Equal(t, 19, s.pos) + } +} + +func TestPlanSamples_RejectsOnlyBeyondTheCeiling(t *testing.T) { + tables := make([]sampleTable, 1500) // 1500 last articles of 768 KB are 1.15 GB + for i := range tables { + tables[i] = bigTable(20, false) } - _, err := planSamples(tables, 59, 1<<30) + _, err := planSamples(tables, 5000, 1<<30) require.ErrorIs(t, err, errTooManyTables) - _, err = planSamples(tables, 600, 100) // bytes + _, err = planSamples(tables[:1000], 5000, 1<<30) + require.NoError(t, err) + _, err = planSamples(tables[:100], 50, 1<<30) // article ceiling require.ErrorIs(t, err, errTooManyTables) } +// When the budget covers the last articles but not everything, the tables that +// also get their first and an interior article are chosen at random. +func TestPlanSamples_OptionalSamplesGoToRandomTables(t *testing.T) { + tables := make([]sampleTable, 100) + for i := range tables { + tables[i] = bigTable(120, false) // 100 x 92 MB: a 9.2 GB release, 1% = 92 MB + } + chosen := map[int]int{} + for range 30 { + plan, err := planSamples(tables, 5000, 1<<30) + require.NoError(t, err) + n, bytes := planBytes(tables, plan) + assert.GreaterOrEqual(t, n, 100) + assert.LessOrEqual(t, bytes, int64(92_160_000), "within 1 percent of the release") + for _, s := range plan { + if s.pos == 0 { + chosen[s.table]++ + } + } + } + assert.Greater(t, len(chosen), 60, "which tables get the optional articles is random, so every table is eventually covered") +} + // Two tables that start with the same article are two tables (they used to // collapse into one when keyed by their first message-id). func TestTablesOf_KeysByTheWholeTable(t *testing.T) { @@ -178,40 +248,79 @@ func TestTablesOf_KeysByTheWholeTable(t *testing.T) { } // The cost of the accept-time header sampling for typical release shapes (768 KB -// articles). Logged for the documentation; the bounds are asserted. +// articles, 100 MB volumes). Logged for the documentation; the bounds are +// asserted. func TestSamplingCostByReleaseShape(t *testing.T) { - pol := DefaultPolicy() - table := func(n int, main bool) sampleTable { - tb := mkTable(n, main) - for i := range tb.sizes { - tb.sizes[i] = 768_000 - } - return tb - } - volumes := func(n int) []sampleTable { + volumes := func(n, articles int) []sampleTable { out := make([]sampleTable, n) for i := range out { - out[i] = table(140, false) // a 100 MB volume + out[i] = bigTable(articles, false) } return out } for name, tc := range map[string]struct { - tables []sampleTable - wantMax int + tables []sampleTable + maxBytes int64 }{ - "1 file (1.1 GB mkv)": {[]sampleTable{table(1500, true)}, 5}, - "50 volumes": {volumes(50), 150}, - "200 volumes": {volumes(200), 600}, + "1 file (1.1 GB mkv)": {[]sampleTable{bigTable(1500, true)}, 8 << 20}, + "50 volumes (100 MB)": {volumes(50, 140), 70_000_000}, + "200 volumes (100 MB)": {volumes(200, 140), 220_000_000}, + "300 volumes (50 MB)": {volumes(300, 70), 250 << 20}, + "500 volumes (100 MB)": {volumes(500, 140), 540_000_000}, } { - plan, err := planSamples(tc.tables, pol.YencSamples(), pol.YencBytes()) + plan, err := planSamples(tc.tables, 2000, 1<<30) require.NoError(t, err, name) - var bytes int64 - for _, s := range plan { - bytes += tc.tables[s.table].sizes[s.pos] + n, bytes := planBytes(tc.tables, plan) + t.Logf("%s: %d articles, %.0f MB read", name, n, float64(bytes)/1e6) + assert.LessOrEqual(t, bytes, tc.maxBytes, name) + } +} + +// slowHeaders answers instantly for the articles in fast and blocks the rest +// until the sampling time is over. +type slowHeaders struct { + SegmentHeaders + fast map[string]bool +} + +func (s slowHeaders) SegmentHeader(ctx context.Context, id string) (YencHeader, error) { + if s.fast[id] { + return s.SegmentHeaders.SegmentHeader(ctx, id) + } + <-ctx.Done() + return YencHeader{}, ctx.Err() +} + +// The sampling is capped by time, not rejected: optional samples that do not +// answer in time are dropped, while every mandatory one still has to answer. +func TestAccept_OptionalSamplesRunningOutOfTimeAreDropped(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("slow", fileSpec{name: "main.mkv", plain: mkvPlain(512), segSize: 128}, fileSpec{name: "side.mkv", plain: mkvPlain(256), segSize: 64}) + fast := map[string]bool{} + var sideLast string + for _, f := range b.Files { + last := b.Pool[f.Segments[len(f.Segments)-1].Ref] + if f.Name == "side.mkv" { + sideLast = last + } + if f.Name == "main.mkv" { + for _, s := range f.Segments { + fast[b.Pool[s.Ref]] = true // everything of the main file + } } - t.Logf("%s: %d articles, %.0f MB read", name, len(plan), float64(bytes)/1e6) - assert.LessOrEqual(t, len(plan), tc.wantMax, name) } - _, err := planSamples(volumes(201), pol.YencSamples(), pol.YencBytes()) - require.ErrorIs(t, err, errTooManyTables, "201 volumes exceed the budget and are rejected, not skipped") + fast[sideLast] = true // only the last article of the other file + e.imp.cfg.Headers = slowHeaders{SegmentHeaders: e.imp.cfg.Headers, fast: fast} + e.imp.cfg.Policy.YencTimeout = 200 * time.Millisecond + _, err := e.imp.Accept(context.Background(), e.req(b)) + require.NoError(t, err, "%v", err) + + delete(fast, sideLast) // now a mandatory sample does not answer + e2 := newEnv(t) + b2 := e2.mkBundle("slow2", fileSpec{name: "main.mkv", plain: mkvPlain(512), segSize: 128}) + e2.imp.cfg.Headers = slowHeaders{SegmentHeaders: e2.imp.cfg.Headers, fast: map[string]bool{}} + e2.imp.cfg.Policy.YencTimeout = 100 * time.Millisecond + _, err = e2.imp.Accept(context.Background(), e2.req(b2)) + require.Error(t, err) + assert.Equal(t, ReasonProbeInconclusive, ReasonOf(err)) } diff --git a/internal/peernet/service/status_levels_test.go b/internal/peernet/service/status_levels_test.go new file mode 100644 index 000000000..08ddbb1ec --- /dev/null +++ b/internal/peernet/service/status_levels_test.go @@ -0,0 +1,35 @@ +package service + +import ( + "context" + "testing" + + "github.com/kipsilabs/altmount/internal/peernet/netid" +) + +// The live run had the open network at level 0 and a private network at level +// 3: the status must report each network's own level (the open one from the +// configuration, the private one from its row). +func TestStatusReportsEachNetworksOwnLevel(t *testing.T) { + s := enabledSettings() + s.OpenLevel, s.AcceptIncoming = 0, true + i := newInstance(t, s) + i.start(t) + ctx := context.Background() + nw, err := i.svc.CreateNetwork(ctx, "home") + if err != nil { + t.Fatal(err) + } + id, _ := netid.ParseNetworkID(nw.ID) + if _, err := i.svc.UpdateNetwork(ctx, id, NetworkUpdate{Level: ptr(3)}); err != nil { + t.Fatal(err) + } + st := i.statusOf(t) + levels := map[string]int{} + for _, n := range st.Networks { + levels[n.Kind] = n.Level + } + if levels[KindOpen] != 0 || levels[KindPrivate] != 3 { + t.Fatalf("levels by kind = %v", levels) + } +} From 0d12d0224e1483e84f0101b4e928254f33ac7955 Mon Sep 17 00:00:00 2001 From: javi11 Date: Sat, 14 Nov 2026 20:53:20 +0100 Subject: [PATCH 4/5] feat(peernet): share files stored in multi-volume archive sets A file stored across several archive volumes is read as one slice per volume. Bundles can now describe that layout explicitly, so the most common release shape can be built, served and accepted. - optional per-file part list in the bundle; absent means one posted file, so existing bundles keep their bytes and IDs - partial ranges are allowed only at part boundaries; each part is a run inside one NZB entry - each sliced part is bound to its volume's own file header (data start, packed size, continuation flags, RAR 5 volume order); unbound slices need a real structural media check - article checks run per part, and the last article of every part is always checked - slices not backed by an archive header stay rejected --- docs/docs/3. Configuration/peer-network.md | 7 +- .../specs/2026-11-12-peer-network-design.md | 25 + .../components/config/peernet/peernetStats.ts | 1 + frontend/tests/peernetStats.test.ts | 1 + internal/importer/peer_bundle_parts_test.go | 96 ++ internal/peernet/archivehdr/archivehdr.go | 17 +- .../peernet/archivehdr/archivehdr_test.go | 33 + .../archivehdr/archivetest/archivetest.go | 20 +- internal/peernet/archivehdr/fuzz_test.go | 7 +- internal/peernet/archivehdr/rar.go | 30 + internal/peernet/bundle/build.go | 83 ++ internal/peernet/bundle/build_parts_test.go | 242 ++++++ internal/peernet/bundle/fuzz_test.go | 12 +- internal/peernet/bundle/helpers_test.go | 1 + internal/peernet/bundle/limits.go | 4 + internal/peernet/bundle/parts.go | 33 + internal/peernet/bundle/parts_func_test.go | 20 + internal/peernet/bundle/parts_test.go | 416 +++++++++ internal/peernet/bundle/types.go | 5 + internal/peernet/bundle/validate.go | 209 ++++- internal/peernet/bundle/wire.go | 17 +- internal/peernet/bundle/wire_decode.go | 39 +- internal/peernet/bundleimport/accept.go | 2 + internal/peernet/bundleimport/cost.go | 12 + internal/peernet/bundleimport/extents.go | 5 + internal/peernet/bundleimport/parts.go | 366 ++++++++ internal/peernet/bundleimport/parts_test.go | 822 ++++++++++++++++++ internal/peernet/bundleimport/sample.go | 148 +++- internal/peernet/bundleimport/types.go | 15 + .../peernet/bundleimport/verify_headers.go | 69 +- internal/peernet/bundleimport/yenc_test.go | 59 ++ internal/peernet/bundlex/builder_test.go | 75 ++ 32 files changed, 2828 insertions(+), 63 deletions(-) create mode 100644 internal/importer/peer_bundle_parts_test.go create mode 100644 internal/peernet/bundle/build_parts_test.go create mode 100644 internal/peernet/bundle/parts.go create mode 100644 internal/peernet/bundle/parts_func_test.go create mode 100644 internal/peernet/bundle/parts_test.go create mode 100644 internal/peernet/bundleimport/parts.go create mode 100644 internal/peernet/bundleimport/parts_test.go diff --git a/docs/docs/3. Configuration/peer-network.md b/docs/docs/3. Configuration/peer-network.md index cb8afc918..733ee78c3 100644 --- a/docs/docs/3. Configuration/peer-network.md +++ b/docs/docs/3. Configuration/peer-network.md @@ -150,6 +150,8 @@ A release is shared as a bundle only when AltMount can prove that nothing privat - releases with **known holes**, unresolved segments or inconsistent local metadata; - **keyed releases** (encrypted RAR or 7-Zip archives) whose password did not come with the NZB's own content. AltMount records, for **new imports only**, whether an archive password came from a `` that was already inside the NZB as it was received. A password typed into AltMount (the link `&p=` or the password field), spelled in the NZB's file name or in a subject line, found inside another release's archive, or supplied before this was recorded does not count, and the release is then not shared. When it does count, the bundle carries the archive key and IV (a receiver needs only those to play the release) and not the password text, which AltMount never stores. +A video stored in a **multi-volume archive set** (`part01.rar` ... `partNN.rar`, RAR 4 or RAR 5 stored, or a split stored 7-Zip set) is shared like any other release. The bundle lists the file as one slice per volume (a "multi-part" layout), recovered from your NZB's file list, so nothing about it needs a re-import. Two cases are counted as not shareable (`part_layout_unrecoverable`): a local PAR2 patch left a repeated article inside the file, or one volume contributes two separate runs. An old-format import with no stored NZB list is shared in the "files" form, where the volume boundaries are recovered from the byte ranges: that is exact for RAR sets and for split 7-Zip sets whose last articles are shorter than the others. + ### What receiving a bundle does At level 1 and above, importing an NZB first asks the network for a bundle of exactly that release (the Release ID of the NZB's segment list), with a short time limit, and falls back to a normal import on any problem. The Stremio flow below imports a release the same way. A received bundle is untrusted. AltMount: @@ -175,7 +177,7 @@ The release identity the bundle proved is kept for the imported files, so they c Before anything is written the layout is also checked against the articles themselves and against the NZB: - **No gaps.** A bundle file may not contain a hole; a release with known holes is not shareable and a received bundle with one is refused. -- **Coverage.** In a bundle for an NZB, every file (and every archive volume or PAR2 file) must be exactly one NZB file entry's segments, in NZB order; no segment is used twice and none is left out. +- **Coverage.** In a bundle for an NZB, every file (and every archive volume or PAR2 file) must be exactly one NZB file entry's segments, in NZB order; no segment is used twice and none is left out. A **multi-part** file (a video stored across archive volumes) is the exception that is spelled out: it lists its parts, each part must be a contiguous run, in article order, of one NZB entry; one entry may give a file at most one part; an entry used by a multi-part file is never also read whole, as an outer volume or as a PAR2 file; and two files stored in the same volume may share only the article where one ends and the next begins (each at the edge of its part, with usable byte ranges that do not overlap). Only the first segment of a part may start inside its article and only its last may end inside it. - **Sizes against your NZB.** When you import an NZB, each declared decoded segment size must fit the NZB's (encoded, a few percent larger) size. - **Sampled article headers.** Articles of each file and archive volume are read from your providers (which ones is described under "Sampling" below) and their yEnc headers must agree with the layout: decoded size, start offset, and total file size. A mismatch refuses the bundle and counts against the publisher. - **Media structure.** Besides the start of each media file, its container index is read at several positions including the end: Matroska/WebM (EBML header, Segment size, and every SeekHead entry such as the Cues), MP4/MOV (the whole top-level box chain must end exactly at the end of the file, or, for fragmented files with thousands of boxes, a fragment box must end exactly there), MPEG-TS/M2TS (sync bytes on the 188, 192 or 204 byte grid at the start, quarters and end). Files muxed for streaming that have no Matroska SeekHead, or an unknown-size Segment, are accepted when a level-1 element opens the Segment, a known Segment size lands on the end of the file and the quarters and the end can be read; trailing free, skip or uuid boxes, a zero-sized last box, 64-bit box sizes and up to 64 KiB of zero padding in MP4 are accepted; other containers get full-length reads at the quarters and end. Every written media file passes this, not only the largest. @@ -185,8 +187,9 @@ Before anything is written the layout is also checked against the articles thems - **Every layout, every network.** Archive layouts (outer volumes read through extents), AES-keyed files, multi-clip files and every media container are accepted from the open network exactly as from a private one, provided they pass the checks below. There is no `layout_not_allowed_open` or `container_not_verifiable` rejection any more. Generic containers (AVI, MPEG and the like) get the head signature and readable-windows check wherever the bundle came from. - **Archive extent checks.** For every file read through outer archive volumes, each extent (outer volume, offset, length) is compared with the archive's own file header, read from your providers through the same hidden reader that will serve the file (so an encrypted outer volume is read decrypted with the bundle's key). The header that precedes the extent's data must be a stored (not compressed) file entry, not a service block or a directory and not itself encrypted; its data must start exactly at the extent's offset and be as long as the extent in that volume; the entry's unpacked size must equal the file's size; and its continuation flags must fit the extent's place (first part, middle part, last part). RAR 4, RAR 5 and 7z (plain end header) are understood. When the outer volume is itself a stored file of a wrapping archive and the volume begins in that archive's first article, the wrapping header is checked as well (stored, same size, and for RAR 5 the IV in the header must equal the bundle's). Up to 64 extents are checked (the first and last of the main media file always, the rest chosen at random). A contradiction is attributable to the publisher: the bundle is refused (`extent_mismatch`), the publisher is reported (fetch back-off only) and the layout goes on the negative cache for 7 days. +- **Multi-part files (archive volumes).** Each slice of a file stored across volumes is bound the same way: the volume is read from its first article, and a stored, unencrypted (or, for a keyed file, encrypted) file entry's data must start at the byte where the slice starts, be as long as the slice, belong to a file of the declared size, and carry continuation flags that fit the slice's place (first, middle, last). For RAR 5 the volume numbers must also run in the order of the slices, so two middle volumes cannot be swapped. A slice that claims to be a whole posted file of media data must not begin like an archive volume. A second file stored after the first in a volume is bound too when your NZB is at hand (the articles before it are taken to be as long as the first one, which its own offset has to confirm). Up to 64 slices are checked (the first and last of every media file always, the rest chosen at random). A contradiction is attributable: `extent_mismatch`, the publisher is reported, the layout goes on the negative cache. A slice that cannot be judged (a second file in a volume of an old-format import, a split 7-Zip set whose end header is in another volume, encoded 7z headers) is unverifiable and treated like an unverifiable extent below. - **What stays unverifiable.** An extent whose archive cannot be judged is *unverifiable*: an unknown format (a disc image such as a Blu-ray ISO), a 7z end header that is compressed or encrypted, encrypted archive headers, a header walk that exceeds its budget, a damaged header chain, or a missing article. Such a file is accepted only when its media passes the structural check (Matroska, MP4 or transport stream) through the layout; a generic container in that situation is refused (`extent_unverifiable`), because readable windows prove nothing about misplaced bytes. Extents beyond the 64 checked are covered by sampling only. Files stored inline in a single archive volume (no outer volumes) are not extents and are verified by the head and structure checks only. -- **Sampling.** The last article of every run of articles (volume or file) is always read: a size lie anywhere in a run shifts every later offset, so the last article's offset and total expose it. The main media file's runs also get their first article and random interior ones (one per 500 articles, at least three). The first and a random interior article of the other runs are read as far as the budget allows, at randomly chosen runs. The budget is one percent of the release size, at least 64 MiB and at most 1 GiB, shared with the archive header reads above and raised if the last articles alone need more; only a release whose last articles exceed 1 GiB (about 1,400 volumes) is refused (`too_many_tables`). Optional samples that run out of the time limit are dropped, mandatory ones are not. A single movie file reads about 5 articles (4 MB), a 50-volume release about 90 (67 MB), a 200-volume release about 280 (215 MB), a 500-volume release about 700 (540 MB), plus a few articles per checked archive volume. Two lies that cancel out inside a run are found only when a sampled article lies between them. With your NZB at hand the first article's yEnc file name must also match the NZB entry it belongs to. The forced full check reads a further random sample of headers (1 percent of the articles, 20 to 500) and fails the import on a contradiction. It samples; it does not prove. +- **Sampling.** The last article of every run of articles (volume or file) is always read: a size lie anywhere in a run shifts every later offset, so the last article's offset and total expose it. The main media file's runs also get their first article and random interior ones (one per 500 articles, at least three). The first and a random interior article of the other runs are read as far as the budget allows, at randomly chosen runs. The budget is one percent of the release size, at least 64 MiB and at most 1 GiB, shared with the archive header reads above and raised if the last articles alone need more; only a release whose last articles exceed 1 GiB (about 1,400 volumes) is refused (`too_many_tables`). Optional samples that run out of the time limit are dropped, mandatory ones are not. A single movie file reads about 5 articles (4 MB), a 50-volume release about 90 (67 MB), a 200-volume release about 280 (215 MB), a 500-volume release about 700 (540 MB), plus a few articles per checked archive volume. A movie stored in a multi-volume archive set reads the last article of every volume, the first article of every checked volume, one article per checked volume for its header and one interior article per 500 articles of the movie: about 165 articles (127 MB) for 50 volumes of 100 MB and about 390 articles (300 MB) for 200 volumes. Two lies that cancel out inside a run are found only when a sampled article lies between them. With your NZB at hand the first article's yEnc file name must also match the NZB entry it belongs to. The forced full check reads a further random sample of headers (1 percent of the articles, 20 to 500) and fails the import on a contradiction. It samples; it does not prove. - **Media structure.** Besides the end of the file, the container is read at random positions inside the file (Matroska: three random 1 MiB windows must contain cluster data; MPEG-TS: eight random packet positions must be in sync; fragmented MP4 must end in a real `moof`+`mdat` pair or `mfra`). What remains unprovable: bytes inside MP4 `mdat`, inside a Matroska frame or between sampled positions. - **Degraded means failed.** A peer release whose full check finds the media playable but with missing segments is failed and removed, never announced. When you supplied the NZB the queue item is not marked failed: it is imported normally from your NZB, once, so you get what a normal import would have produced. A release imported directly (Stremio peer play) is failed. A missing article alone does not count against the publisher (it can be propagation just after posting); only a contradiction between the layout and the articles' own headers does, and that layout is then ignored whoever offers it, for 7 days. - **Corroboration** counts distinct /24 (IPv4) or /48 (IPv6) ranges of the address this node actually connected to when it last verified the publisher, never host names; with the open-network switch off, open-network publishers are not even contacted. diff --git a/docs/superpowers/specs/2026-11-12-peer-network-design.md b/docs/superpowers/specs/2026-11-12-peer-network-design.md index 889765bbb..3aa48ced2 100644 --- a/docs/superpowers/specs/2026-11-12-peer-network-design.md +++ b/docs/superpowers/specs/2026-11-12-peer-network-design.md @@ -1019,3 +1019,28 @@ Owner decision: every kind of release is accepted on every network. The second r **State and reasons.** Removed reasons: `layout_not_allowed_open`, `container_not_verifiable`. New: `extent_mismatch`, `extent_unverifiable`. `too_many_tables` now means the ceiling is exceeded. **Activity card.** On a node with the open network at level 0 and a private network at level 3 the status carries `level: 0` for the open network (from the configuration) and `level: 3` for the private one (from its row), both under the same field name, and the card's rules (`some(level >= 1)`, `some(level >= 3)`) evaluate correctly on that shape; a test pins it in the service and in the frontend. No defect in the derivation was found; the non-zero-counter fallback stays, because the serving counters are since-start counters and stay non-zero after a level is lowered. + +## Phase 3: multi-part inline layout + +Owner requirement: every kind of release is shareable and accepted. The third hardening round left one gap: a stored multi-volume archive set (RAR 4, RAR 5 or split 7z, no outer encryption) is flattened by the importer into one inline table with partial ranges at every volume boundary, which the interior-range rule refused and the one-NZB-entry coverage rule could not express. This section adds the layout that expresses it exactly, without relaxing either protection. + +**Schema.** `File` gets field 12, `parts`: a packed list of varints, the number of segments in each part. It is written after `file_id` (field 11) and only when set, so every bundle that has no multi-part file encodes to exactly the same bytes as before (pinned by digests of the two reference bundles taken from the encoder before the change; schema version stays 1). Absent means the table is read as one posted file, as before. Present means the table is the concatenation of slices: one part per archive volume (or, with one part, a slice of the middle of one volume such as a second file stored after the first). Canonical form: never empty, every length at least 1, lengths add up to the segment count, at most 10,000 per file and 50,000 per bundle (counted from the packed bytes before anything is allocated), only on a file with an inline table. Decode then re-encode still has to be the identity; fuzz seeds and the mutation fuzzer include multi-part bundles. + +**Validation rules (all schemes).** Inside a part the table rules hold: only the part's first segment may start inside its article and only its last may end inside it; a part boundary is the only place a usable range may skip bytes, which keeps the interior-range protection (the 100-articles-10-bytes shape is still refused as one table and as one part). A pool entry is used once within the file. Two parts of different files may share a pool entry only when it is at the edge of every part that uses it (a part's first or last segment) and the usable ranges are pairwise disjoint: two files stored back to back in one volume share the article where one ends and the next begins, and so can several tiny files in one article. Any other use of an entry by two tables is refused, as in the second round; an entry used by a multi-part file is never also used by a table that is not (whole inline file, outer volume, PAR2 file). + +**Coverage rule (nzb scheme), exactly.** For a file without `parts`: unchanged (the table is exactly one NZB entry's segments in order). For a file with `parts`: each part is a run of consecutive pool positions, in order, entirely inside one NZB entry (not necessarily the whole entry); an entry supplies at most one part to a file; the order of parts is the order of the file's bytes and is not required to follow the NZB's order (an obfuscated NZB lists volumes in any order), which is why the receiver binds it (below). The same entry may supply parts to different files only through the shared-article rule above. Everything else is `ErrCoverage`. + +**Accept-time checks.** +- yEnc, per part. A part that is a whole NZB entry (nzb scheme), or whose ranges are full (files scheme), claims to be a whole posted file: first article at offset 0, posted total equal to the sum of the part, as for any table. Any other part is a slice: every sampled article must agree on where the part begins in its posted file and on the posted total (`relState`), the last article of every part is always sampled, and a part that is checked against an archive header has its first article sampled too. Name binding against the NZB subject works per entry. A file without `parts` behaves exactly as before. +- Header binding. For each slice (up to 64 per bundle: the first and last of every media file, the rest by `crypto/rand`), the volume is read from its first article through the staged reader and the archive's headers are scanned (`archivehdr`): a stored file entry whose data area starts at the byte where the part starts, whose packed size is the part's length, whose unpacked size is the file's size, whose continuation flags fit the part's place (first, middle, last), and for a keyed file that is encrypted (the first part's IV must equal the bundle's). RAR 5 volume numbers (`Volume.VolumeNumber`) must run in the order of the parts, which catches a swap of two middle volumes that no flag can; RAR 4 and 7z declare none, so that swap is not detectable for them. A slice that does not begin at its posted file's first article is read through a view that starts at the volume's first article, in the nzb scheme only (the articles before it are taken to be as long as the first, which the part's own offset must confirm); in the files scheme it is unverifiable. A full-range part of a media file that claims to be a whole posted file is checked not to begin like an archive volume (a whole RAR volume declared as raw media data is refused). +- Mismatch is attributable: `extent_mismatch`, publisher report, negative cache. Unverifiable (unknown or encoded archive, a 7z end header in another volume, a second file of an old-format import, a spent budget, a damaged chain): the file is accepted only when its real structural media check passes, and a generic container is refused (`extent_unverifiable`), as for extents. +- One budget. The header reads share the accept-wide budget with the yEnc samples and the extent checks; the first article of every checked part is a mandatory sample, the interior samples of the parts of the main media file are planned over all of them as one table (one per 500 articles of the file, at least three). +- Forced full check. The stored metadata is flat, so the volume boundaries are recovered from the ranges (a segment that starts inside its article, one that ends before its article does, one longer than the one before it) and each recovered piece is checked relative to where its first sampled article begins. A file with no recovered boundary is checked as one posted file, as before. + +**Builder.** `FromLocal` recovers the parts deterministically. Nzb scheme: a part is a maximal run of consecutive articles of one NZB entry (the list says which entry every article belongs to). One run that is a whole entry is the original layout and writes no `parts`; anything else writes them. An entry that supplies two runs (a repeated article from a local patch, a skipped article) is `part_layout_unrecoverable`. Files scheme (old-format imports, no NZB list): the boundary rule is a segment i>0 that starts inside its article, or a segment that ends before its article does, or a segment longer than the one before it. For RAR 4 and RAR 5 the first of these is exact (every volume begins with a header). For a split 7z set the interior boundaries leave full ranges on both sides and are visible only through the size rule, which fails when the last article of a volume is exactly as long as the others (the volume size is a multiple of the article size); such a file is then built as one posted file and a receiver refuses it (attributable to nobody the builder can detect). A second file stored after the first in an old-format import is built as one table too, since a lone slice cannot be told from a whole file without the NZB list. The library has not been measured for how often either case occurs. + +**`ToLocal`.** Parts are flattened back to the segment list with partial ranges that the readers expect, field by field equal to what the importer wrote (tested with the importer's real RAR and 7z fixtures, 1, 2 and 14 volumes). + +**What remains unprovable.** A slice that cannot be bound (a second media file in a volume of an old-format import, a split 7z whose end header is elsewhere) is accepted on its media structure alone, so bytes between the structurally checked positions are not proven. Two swapped middle volumes of a RAR 4 or 7z set are not detected. Parts beyond the 64 checked are covered by sampling only. A part chosen to hide in an unchecked slot can still be forged; the random choice makes that a gamble per forged part. + +**Reasons.** New not-shareable reason `part_layout_unrecoverable`. No new accept reason: `extent_mismatch`, `extent_unverifiable` and `yenc_mismatch` are used. diff --git a/frontend/src/components/config/peernet/peernetStats.ts b/frontend/src/components/config/peernet/peernetStats.ts index 9852c3d96..5a0934263 100644 --- a/frontend/src/components/config/peernet/peernetStats.ts +++ b/frontend/src/components/config/peernet/peernetStats.ts @@ -51,6 +51,7 @@ const NOT_SHAREABLE_LABELS: ReadonlyMap = new Map([ ["unsupported_encryption", "Uses an encryption type that cannot be shared"], ["password_not_from_nzb", "Password did not come inside the NZB"], ["healed_by_local_patch", "Repaired with local PAR2 patches"], + ["part_layout_unrecoverable", "Archive volumes could not be told apart reliably"], ["known_holes", "Has known missing parts"], ["patch_status_unknown", "Repair status was not known"], ["no_nzb_segment_list", "No stored NZB segment list"], diff --git a/frontend/tests/peernetStats.test.ts b/frontend/tests/peernetStats.test.ts index a1b93d034..6164d2615 100644 --- a/frontend/tests/peernetStats.test.ts +++ b/frontend/tests/peernetStats.test.ts @@ -76,6 +76,7 @@ const BACKEND_NOT_SHAREABLE_KEYS = [ "unsupported_encryption", "password_not_from_nzb", "healed_by_local_patch", + "part_layout_unrecoverable", "known_holes", "patch_status_unknown", "no_nzb_segment_list", diff --git a/internal/importer/peer_bundle_parts_test.go b/internal/importer/peer_bundle_parts_test.go new file mode 100644 index 000000000..a84e9e650 --- /dev/null +++ b/internal/importer/peer_bundle_parts_test.go @@ -0,0 +1,96 @@ +package importer + +import ( + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/testsupport/nzbbuild" +) + +// A file the importer reads out of a multi-volume stored archive set is one flat +// segment list with partial ranges at every volume boundary. The bundle builder +// recovers the volumes from the NZB entries, and the receiving side converts the +// parts back to exactly that list. +func TestPeerBundle_ImporterMultiVolumeFileRoundTripsThroughParts(t *testing.T) { + for _, tc := range []struct { + name string + fixture string + pattern string + inner string + // parts is the number of parts expected: 0 for a set of one volume, which + // keeps the original single-table layout. + parts int + }{ + {"rar two volumes", "rar_multi", "archive.part*.rar", "", 2}, + {"rar twelve volumes", "rar_widthmismatch", "archive.part*.rar", "", 0}, + {"7z two volumes", "7z_multi", "archive.7z.*", "", 2}, + {"rar one volume", "rar_single", "archive.rar", "", 0}, + {"7z one volume", "7z_single", "archive.7z", "", 0}, + } { + t.Run(tc.name, func(t *testing.T) { + env := newBatteryEnv(t) + var files []nzbbuild.File + var ids [][]string + matches, err := filepath.Glob(filepath.Join("testdata", tc.fixture, tc.pattern)) + require.NoError(t, err) + sort.Strings(matches) + require.NotEmpty(t, matches) + if tc.fixture == "rar_widthmismatch" { + tc.parts = len(matches) + } + for i, m := range matches { + v := filepath.Base(m) + content := loadFixture(t, filepath.Join(tc.fixture, v)) + segs := env.registerContent("pp-"+strings.ReplaceAll(tc.name, " ", "-")+string(rune('a'+i)), content, archivePartSize, 1.0, nil) + files = append(files, nzbbuild.File{Subject: v, Segments: segs}) + var list []string + for _, s := range segs { + list = append(list, s.ID) + } + ids = append(ids, list) + } + _, _, err = env.runImport(nzbbuild.Build(files...), "archive") + require.NoError(t, err) + names := env.listDir("/archive") + require.Len(t, names, 1) + tc.inner = names[0] + meta := env.readMeta("/archive/" + tc.inner) + require.NotEmpty(t, meta.SegmentData) + + in := bundle.ReleaseInput{Scheme: bundle.SchemeNZB, Title: "Archive Release", PostedAt: time.Unix(1_700_000_000, 0), NZBFiles: ids, + Files: []bundle.FileInput{{Name: tc.inner, Meta: meta}}} + bd, err := bundle.FromLocal(in) + require.NoError(t, err) + require.Len(t, bd.Files, 1) + if tc.parts == 0 { + assert.Empty(t, bd.Files[0].Parts, "a single volume keeps the original layout") + } else { + assert.Len(t, bd.Files[0].Parts, tc.parts, "one part per volume") + } + + wire, id, err := bundle.Pack(bd) + require.NoError(t, err) + got, err := bundle.Verify(wire, id, bd.ReleaseID) + require.NoError(t, err) + loc, err := bundle.ToLocal(got, time.Unix(1_700_000_100, 0)) + require.NoError(t, err) + have := loc.Files[0].Meta + + assert.Equal(t, meta.FileSize, have.FileSize) + require.Len(t, have.SegmentData, len(meta.SegmentData)) + for i, want := range meta.SegmentData { + assert.Equal(t, want.Id, have.SegmentData[i].Id, "segment %d", i) + assert.Equal(t, want.SegmentSize, have.SegmentData[i].SegmentSize, "segment %d", i) + assert.Equal(t, want.StartOffset, have.SegmentData[i].StartOffset, "segment %d", i) + assert.Equal(t, want.EndOffset, have.SegmentData[i].EndOffset, "segment %d", i) + } + }) + } +} diff --git a/internal/peernet/archivehdr/archivehdr.go b/internal/peernet/archivehdr/archivehdr.go index a8a88ee92..f652cc5e1 100644 --- a/internal/peernet/archivehdr/archivehdr.go +++ b/internal/peernet/archivehdr/archivehdr.go @@ -22,6 +22,7 @@ package archivehdr import ( + "bytes" "errors" "fmt" "io" @@ -127,6 +128,10 @@ type Volume struct { Format Format // Size is the number of bytes of the volume. Size int64 + // VolumeNumber is the 0-based volume number a RAR 5 main header declares + // (the first volume of a set carries none and is 0); -1 when the format + // does not say (RAR 4, 7z, or a RAR 5 archive that is not a volume). + VolumeNumber int64 // Blocks are the data blocks found in order, up to the offset asked for. Blocks []Block // Reads is the number of reads made. @@ -203,7 +208,7 @@ func Scan(r io.ReaderAt, size int64, upTo int64, lim Limits) (*Volume, error) { } return nil, err } - v := &Volume{Size: size} + v := &Volume{Size: size, VolumeNumber: -1} switch { case len(head) >= 8 && string(head[:8]) == string(rar5Sig): v.Format = FormatRAR5 @@ -258,6 +263,10 @@ type Extent struct { Index, Count int // FileSize is the size of the whole file the extents make up. FileSize int64 + // Keyed says the layout carries an AES key for the file: its entry must be + // encrypted, and when both IVs are known they must be equal. + Keyed bool + IV []byte } // Check judges an extent against a scan of its volume. A scan that stopped on @@ -303,8 +312,12 @@ func checkBlock(v *Volume, b *Block, e Extent) Result { return bad("the data area at %d belongs to a service block or a directory, not a file", e.Offset) case !b.Stored: return bad("the file at %d is compressed, not stored", e.Offset) - case b.Encrypted: + case b.Encrypted && !e.Keyed: return bad("the file at %d is itself encrypted but the layout carries no key for it", e.Offset) + case e.Keyed && !b.Encrypted: + return bad("the layout carries a key for the file at %d but the entry is not encrypted", e.Offset) + case e.Keyed && b.IV != nil && e.IV != nil && !bytes.Equal(b.IV, e.IV): + return bad("the IV of the entry at %d differs from the layout's", e.Offset) case b.DataSize != e.Length: return bad("the header says %d bytes in this volume, the layout %d", b.DataSize, e.Length) case b.UnpackedSize >= 0 && b.UnpackedSize != e.FileSize: diff --git a/internal/peernet/archivehdr/archivehdr_test.go b/internal/peernet/archivehdr/archivehdr_test.go index f65f62cad..5b5f13723 100644 --- a/internal/peernet/archivehdr/archivehdr_test.go +++ b/internal/peernet/archivehdr/archivehdr_test.go @@ -292,3 +292,36 @@ func TestRealArchivesFromTheImporterFixtures(t *testing.T) { assert.ErrorIs(t, err, ErrCompressedHeader, "a real 7z whose end header is encoded is unverifiable") } } + +func TestRAR5VolumeNumberIsReported(t *testing.T) { + scan := func(data []byte) *Volume { + v, err := Scan(bytes.NewReader(data), int64(len(data)), int64(len(data)), Limits{}) + require.NoError(t, err) + return v + } + for n := 0; n < 4; n++ { + a := archivetest.RAR5Volume(n, archivetest.File{Name: "movie.mkv", Data: payload(100), SplitBefore: n > 0, SplitAfter: true}) + assert.EqualValues(t, n, scan(a.Data).VolumeNumber, "volume %d", n) + } + assert.EqualValues(t, -1, scan(archivetest.RAR5(archivetest.File{Name: "a", Data: payload(10)}).Data).VolumeNumber, "not a volume") + assert.EqualValues(t, -1, scan(archivetest.RAR4(archivetest.File{Name: "a", Data: payload(10)}).Data).VolumeNumber, "rar4 has none") +} + +func TestKeyedExtentNeedsAnEncryptedEntryWithTheSameIV(t *testing.T) { + iv := bytes.Repeat([]byte{7}, 16) + p := payload(64) + a := archivetest.RAR5(archivetest.File{Name: "movie.mkv", Data: p, IV: iv}) + keyed := func(iv []byte) Extent { + e := ext(a.DataOffsets[0], 64, 0, 1, 64) + e.Keyed, e.IV = true, iv + return e + } + assert.Equal(t, Bound, verify(a.Data, keyed(iv)).Outcome) + assert.Equal(t, Bound, verify(a.Data, keyed(nil)).Outcome, "no IV to compare") + assert.Equal(t, Mismatch, verify(a.Data, keyed(bytes.Repeat([]byte{8}, 16))).Outcome) + assert.Equal(t, Mismatch, verify(a.Data, ext(a.DataOffsets[0], 64, 0, 1, 64)).Outcome, "an encrypted entry needs a keyed extent") + plain := archivetest.RAR5(archivetest.File{Name: "movie.mkv", Data: p}) + e := ext(plain.DataOffsets[0], 64, 0, 1, 64) + e.Keyed = true + assert.Equal(t, Mismatch, verify(plain.Data, e).Outcome, "a keyed extent needs an encrypted entry") +} diff --git a/internal/peernet/archivehdr/archivetest/archivetest.go b/internal/peernet/archivehdr/archivetest/archivetest.go index 8cd68440e..68c43ffda 100644 --- a/internal/peernet/archivehdr/archivetest/archivetest.go +++ b/internal/peernet/archivehdr/archivetest/archivetest.go @@ -51,7 +51,14 @@ func vint(v uint64) []byte { // RAR5 builds a RAR 5 archive: signature, main header, one block per file, end // of archive. -func RAR5(files ...File) Built { +func RAR5(files ...File) Built { return rar5(-1, files) } + +// RAR5Volume builds one volume of a multi-volume RAR 5 set: the main header +// carries the volume flag and, after the first volume, the volume number +// (0-based). +func RAR5Volume(volume int, files ...File) Built { return rar5(volume, files) } + +func rar5(volume int, files []File) Built { var buf bytes.Buffer buf.Write([]byte{'R', 'a', 'r', '!', 0x1A, 0x07, 0x01, 0x00}) block := func(content []byte) { @@ -63,7 +70,16 @@ func RAR5(files ...File) Built { buf.Write(size) buf.Write(content) } - block(append(vint(1), append(vint(0), vint(0)...)...)) // main: type 1, flags 0, archive flags 0 + main := append(vint(1), vint(0)...) // main: type 1, flags 0 + switch { + case volume < 0: + main = append(main, vint(0)...) // archive flags 0 + case volume == 0: + main = append(main, vint(0x1)...) + default: + main = append(append(main, vint(0x3)...), vint(uint64(volume))...) + } + block(main) var offsets []int64 for _, f := range files { typ, flags := uint64(2), uint64(0x2) diff --git a/internal/peernet/archivehdr/fuzz_test.go b/internal/peernet/archivehdr/fuzz_test.go index 60c0cb67f..1308998eb 100644 --- a/internal/peernet/archivehdr/fuzz_test.go +++ b/internal/peernet/archivehdr/fuzz_test.go @@ -17,6 +17,8 @@ func fuzzSeeds() []archivetest.Built { archivetest.RAR5(archivetest.File{Name: "a", Data: p[:40]}, archivetest.File{Name: "m.mkv", Data: p, SplitBefore: true, SplitAfter: true, UnpackedSize: 900}), archivetest.RAR5(archivetest.File{Name: "m.mkv", Data: p, IV: iv}, archivetest.File{Name: "c", Data: p[:9], Service: true}), archivetest.RAR5(archivetest.File{Name: "m.mkv", Data: p, Compressed: true}), + archivetest.RAR5Volume(0, archivetest.File{Name: "m.mkv", Data: p, SplitAfter: true, UnpackedSize: 900}), + archivetest.RAR5Volume(3, archivetest.File{Name: "m.mkv", Data: p, SplitBefore: true, SplitAfter: true, UnpackedSize: 900, IV: iv}), archivetest.RAR4(archivetest.File{Name: "m.mkv", Data: p}), archivetest.RAR4(archivetest.File{Name: "a", Data: p[:40]}, archivetest.File{Name: "m.mkv", Data: p, SplitBefore: true, UnpackedSize: 600}), archivetest.RAR4(archivetest.File{Name: "m.mkv", Data: p, IV: iv, Compressed: true}), @@ -57,7 +59,7 @@ func addSeeds(f *testing.F) { func fuzzVerifyBody(t *testing.T, data []byte, off, length int64, idx, count uint8, fsize int64) { lim := Limits{MaxBlocks: 24, MaxReads: 48, MaxHeader: 4096, Max7zHeader: 4096} - e := Extent{Offset: off, Length: length, Index: int(idx), Count: int(count), FileSize: fsize} + e := Extent{Offset: off, Length: length, Index: int(idx), Count: int(count), FileSize: fsize, Keyed: count&0x80 != 0} cr := &countingReader{data: data} r := Verify(cr, int64(len(data)), e, lim) if cr.reads > lim.MaxReads { @@ -69,6 +71,9 @@ func fuzzVerifyBody(t *testing.T, data []byte, off, length int64, idx, count uin if r.Outcome == Bound && (off < 0 || length < 1 || off > int64(len(data)) || length > int64(len(data))-off) { t.Fatalf("bound an extent [%d,+%d) outside a %d byte volume", off, length, len(data)) } + if v, _ := Scan(bytes.NewReader(data), int64(len(data)), off, lim); v != nil && v.VolumeNumber < -1 { + t.Fatalf("volume number %d", v.VolumeNumber) + } if r2 := Verify(bytes.NewReader(data), int64(len(data)), e, lim); r2 != r { t.Fatalf("not deterministic: %+v then %+v", r, r2) } diff --git a/internal/peernet/archivehdr/rar.go b/internal/peernet/archivehdr/rar.go index cd1fae3a1..61f1361d2 100644 --- a/internal/peernet/archivehdr/rar.go +++ b/internal/peernet/archivehdr/rar.go @@ -117,6 +117,8 @@ func scanRAR5(d *reader, v *Volume, upTo int64, lim Limits) error { return err } switch kind { + case r5Main: + v.VolumeNumber = rar5VolumeNumber(hdr[4+n:]) case r5Encrypt: return ErrEncryptedHeaders case r5End: @@ -183,6 +185,34 @@ func parseRAR5(body []byte, off, total int64) (Block, uint64, error) { return blk, typ, nil } +// rar5VolumeNumber reads the volume number of a RAR 5 main header (body after +// the size field): archive flags bit 0x1 says the archive is a volume, bit 0x2 +// that a volume number follows (the first volume has none and is 0). It returns +// -1 for an archive that is not a volume or a header it cannot read. +func rar5VolumeNumber(body []byte) int64 { + c := &cursor{b: body} + c.vint() // type + flags := c.vint() + if flags&r5FlagExtra != 0 { + c.vint() + } + if flags&r5FlagData != 0 { + c.vint() + } + arch := c.vint() + if c.err || arch&0x1 == 0 { + return -1 + } + if arch&0x2 == 0 { + return 0 + } + n := c.vint() + if c.err || n > 1<<40 { + return -1 + } + return int64(n) +} + // parseRAR5Extra reads the extra records of a file header for the file // encryption record (type 1): version, flags, KDF count, 16-byte salt, 16-byte // IV. diff --git a/internal/peernet/bundle/build.go b/internal/peernet/bundle/build.go index 12a16b9c9..4ef49cdac 100644 --- a/internal/peernet/bundle/build.go +++ b/internal/peernet/bundle/build.go @@ -57,6 +57,10 @@ const ( ReasonOtherEncryption ShareReason = "unsupported_encryption" ReasonPasswordNotNZB ShareReason = "password_not_from_nzb" ReasonLocalPatch ShareReason = "healed_by_local_patch" + // ReasonPartLayout: the file is stored across archive volumes but its part + // boundaries cannot be recovered reliably (a repeated article, or two + // separate runs of one volume). + ReasonPartLayout ShareReason = "part_layout_unrecoverable" ) // NotShareableError reports the first file that makes the release not @@ -154,6 +158,9 @@ type poolBuilder struct { nzb bool idx map[string]int64 pool []string + // nzb scheme: the pool range of every NZB file entry. + entStart []int64 + entLen []int } func (p *poolBuilder) ref(raw string) (int64, error) { @@ -232,6 +239,76 @@ func effectiveOuter(m *metapb.FileMetadata, ns *metapb.NestedSegmentSource) *met return ns } +// entryOf returns the NZB file entry that holds pool position pos (nzb scheme). +func (p *poolBuilder) entryOf(pos int64) int { + i := sort.Search(len(p.entStart), func(k int) bool { return p.entStart[k]+int64(p.entLen[k]) > pos }) + for i < len(p.entStart) && p.entLen[i] == 0 { + i++ + } + return i +} + +// partsOf recovers the part boundaries of an inline table, deterministically. +// It returns nil for a table that is one whole posted file (the original +// layout, which encodes without parts). +// +// Nzb scheme: the NZB list says which entry (archive volume) every article +// belongs to, so a part is a maximal run of consecutive articles of one entry. +// A single run that is a whole entry is the original layout; anything else is a +// multi-part layout, and a layout in which an entry supplies two runs (a +// repeated article from a local patch, a skipped article) is refused. +// +// Files scheme: no entries are known. A boundary is a segment that starts +// inside its article (every volume of a RAR set begins with a header, so this is +// exact for RAR 4 and 5), a segment that ends before its article does, or a +// segment longer than the one before it (a posted file's last article is its +// shortest, which marks the volume boundary of a split 7z set that leaves both +// sides whole). It cannot see a boundary between two volumes that are both whole +// and have equal article sizes; such a file is then built as one posted file, +// which a receiver will not accept. +func (p *poolBuilder) partsOf(name string, segs []Segment) ([]int, error) { + if len(segs) < 2 { + return nil, nil + } + if !p.nzb { + return rangeParts(segs), nil + } + var lens []int + var entries []int + for i, s := range segs { + if i == 0 || s.Ref != segs[i-1].Ref+1 || p.entryOf(s.Ref) != entries[len(entries)-1] { + lens, entries = append(lens, 0), append(entries, p.entryOf(s.Ref)) + } + lens[len(lens)-1]++ + } + seen := make(map[int]struct{}, len(entries)) + for _, e := range entries { + if _, dup := seen[e]; dup { + return nil, &NotShareableError{File: name, Reason: ReasonPartLayout} + } + seen[e] = struct{}{} + } + if len(lens) == 1 && segs[0].Ref == p.entStart[entries[0]] && lens[0] == p.entLen[entries[0]] { + return nil, nil + } + return lens, nil +} + +func rangeParts(segs []Segment) []int { + var lens []int + start := 0 + for i := 1; i < len(segs); i++ { + if segs[i].Start > 0 || segs[i-1].End < segs[i-1].Size-1 || segs[i].Size > segs[i-1].Size { + lens = append(lens, i-start) + start = i + } + } + if len(lens) == 0 { + return nil + } + return append(lens, len(segs)-start) +} + func (p *poolBuilder) file(pf preparedFile) (File, error) { m := pf.in.Meta if (len(m.AesKey) > 0) != (m.Encryption == metapb.Encryption_AES) { @@ -246,6 +323,9 @@ func (p *poolBuilder) file(pf preparedFile) (File, error) { if f.Segments, err = p.table(m.SegmentData); err != nil { return File{}, err } + if f.Parts, err = p.partsOf(pf.name, f.Segments); err != nil { + return File{}, err + } index := map[string]int{} for _, ns := range m.NestedSources { if ns == nil { @@ -298,7 +378,9 @@ func nzbPool(files [][]string) (*poolBuilder, []int, error) { } p := &poolBuilder{nzb: true, idx: map[string]int64{}} counts := make([]int, len(files)) + p.entStart = make([]int64, len(files)) for i, f := range files { + p.entStart[i] = int64(len(p.pool)) for _, raw := range f { if holes.IsPlaceholderID(raw) { continue @@ -317,6 +399,7 @@ func nzbPool(files [][]string) (*poolBuilder, []int, error) { counts[i]++ } } + p.entLen = counts return p, counts, nil } diff --git a/internal/peernet/bundle/build_parts_test.go b/internal/peernet/bundle/build_parts_test.go new file mode 100644 index 000000000..4cf32405a --- /dev/null +++ b/internal/peernet/bundle/build_parts_test.go @@ -0,0 +1,242 @@ +package bundle + +import ( + "errors" + "fmt" + "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// stored mimics what the importer writes for a file stored in a multi-volume +// archive set: the volumes' articles (volumeArticles each), of which the file +// takes [dataStart, volumeBytes-trailer) of every volume, flattened into one +// segment list. +type storedSet struct { + ids [][]string // article ids per volume + sizes [][]int64 // decoded article sizes per volume + segs []specSeg + truthLen []int // true part lengths +} + +func newStoredSet(r *rand.Rand, volumes int, articles func(v int) int, head, tail int64, lastShort bool) storedSet { + var s storedSet + for v := 0; v < volumes; v++ { + n := articles(v) + ids, sizes := make([]string, n), make([]int64, n) + for j := range ids { + ids[j] = fmt.Sprintf("v%d-a%d@set.test", v, j) + sizes[j] = 1000 + } + if lastShort { + sizes[n-1] = int64(200 + r.Intn(700)) + } + s.ids, s.sizes = append(s.ids, ids), append(s.sizes, sizes) + var total int64 + for _, z := range sizes { + total += z + } + lo, hi := head, total-tail // usable bytes [lo, hi) of the volume + var at, count int64 + for j, z := range sizes { + a, b := max(lo-at, 0), min(hi-at, z) + if b > a { + s.segs = append(s.segs, specSeg{id: ids[j], size: z, start: a, end: b - 1}) + count++ + } + at += z + } + s.truthLen = append(s.truthLen, int(count)) + } + return s +} + +func (s storedSet) nzb() [][]string { return s.ids } + +func (s storedSet) file() specFile { + var size int64 + for _, g := range s.segs { + size += g.end - g.start + 1 + } + return specFile{name: "movie.mkv", size: size, date: 1700000100, segs: s.segs} +} + +func buildInput(scheme Scheme, nzb [][]string, files ...specFile) ReleaseInput { + in := ReleaseInput{Scheme: scheme, Title: "Some Release 2024", PostedAt: time.Unix(1700000000, 0), NZBFiles: nzb} + for _, f := range files { + in.Files = append(in.Files, f.input()) + } + return in +} + +func TestFromLocal_MultiVolumeStoredSetBecomesParts(t *testing.T) { + r := rand.New(rand.NewSource(1)) + set := newStoredSet(r, 4, func(int) int { return 6 }, 60, 20, false) + bd, err := FromLocal(buildInput(SchemeNZB, set.nzb(), set.file())) + require.NoError(t, err) + require.Len(t, bd.Files, 1) + assert.Equal(t, set.truthLen, bd.Files[0].Parts) + require.NoError(t, Validate(bd)) +} + +func TestFromLocal_PartsAreDeterministicAndOrderIndependent(t *testing.T) { + r := rand.New(rand.NewSource(2)) + set := newStoredSet(r, 3, func(int) int { return 5 }, 60, 20, false) + nfo := specFile{name: "movie.nfo", size: 2000, date: 1700000100, segs: []specSeg{full("n1@set.test", 1000), full("n2@set.test", 1000)}} + nzb := append(set.nzb(), []string{"n1@set.test", "n2@set.test"}) + a, err := FromLocal(buildInput(SchemeNZB, nzb, set.file(), nfo)) + require.NoError(t, err) + b, err := FromLocal(buildInput(SchemeNZB, nzb, nfo, set.file())) + require.NoError(t, err) + assert.Equal(t, mustEncode(t, a), mustEncode(t, b)) + c, err := FromLocal(buildInput(SchemeNZB, nzb, set.file(), nfo)) + require.NoError(t, err) + assert.Equal(t, mustEncode(t, a), mustEncode(t, c)) +} + +// A whole-entry table (a single-volume file, or a plain file) keeps the +// original layout: no parts, the same bytes as before the layout existed. +func TestFromLocal_WholeEntryTableStaysSinglePart(t *testing.T) { + r := rand.New(rand.NewSource(3)) + one := newStoredSet(r, 1, func(int) int { return 5 }, 60, 20, false) + bd, err := FromLocal(buildInput(SchemeNZB, one.nzb(), one.file())) + require.NoError(t, err) + assert.Empty(t, bd.Files[0].Parts) + bd, err = FromLocal(buildInput(SchemeFiles, nil, one.file())) + require.NoError(t, err) + assert.Empty(t, bd.Files[0].Parts) +} + +func TestFromLocal_SliceOfOneEntryIsASinglePartLayout(t *testing.T) { + // Two files stored back to back in one volume: neither is a whole entry. + ids := []string{"a0@s.test", "a1@s.test", "a2@s.test", "a3@s.test", "a4@s.test"} + a := specFile{name: "a.mkv", size: 1000 - 60 + 1000 + 500, segs: []specSeg{{id: ids[0], size: 1000, start: 60, end: 999}, full(ids[1], 1000), {id: ids[2], size: 1000, start: 0, end: 499}}} + b := specFile{name: "b.mkv", size: 500 + 1000 + 900, segs: []specSeg{{id: ids[2], size: 1000, start: 500, end: 999}, full(ids[3], 1000), {id: ids[4], size: 1000, start: 0, end: 899}}} + bd, err := FromLocal(buildInput(SchemeNZB, [][]string{ids}, a, b)) + require.NoError(t, err) + for _, f := range bd.Files { + assert.Equal(t, []int{3}, f.Parts, f.Name) + } +} + +func TestFromLocal_PartLayoutThatCannotBeRepresentedIsNotShareable(t *testing.T) { + r := rand.New(rand.NewSource(4)) + cases := map[string]func(set *storedSet) specFile{ + "a patched segment repeats an article": func(set *storedSet) specFile { + f := set.file() + last := f.segs[len(f.segs)-1] + f.segs = append(f.segs, specSeg{id: last.id, size: last.size, start: last.start, end: last.start + 9}) + f.size += 10 + return f + }, + "two runs of one volume": func(set *storedSet) specFile { + f := set.file() + f.segs = append(append([]specSeg{}, f.segs[:2]...), f.segs[3:]...) // skip an article of volume 0 + f.size = specFile{segs: f.segs}.fileSize() + return f + }, + } + for name, mut := range cases { + t.Run(name, func(t *testing.T) { + set := newStoredSet(r, 3, func(int) int { return 5 }, 60, 20, false) + _, err := FromLocal(buildInput(SchemeNZB, set.nzb(), mut(&set))) + require.ErrorIs(t, err, ErrNotShareable) + var ns *NotShareableError + require.True(t, errors.As(err, &ns)) + assert.Equal(t, ReasonPartLayout, ns.Reason) + }) + } +} + +func (f specFile) fileSize() int64 { + var n int64 + for _, s := range f.segs { + n += s.end - s.start + 1 + } + return n +} + +// Files scheme: no NZB entries exist, so the boundaries come from the ranges. +// For a RAR set the rule is exact: every volume starts with a header, so the +// first segment of every part after the first starts inside its article. +func TestFromLocal_FilesSchemeBoundariesFromRanges(t *testing.T) { + r := rand.New(rand.NewSource(5)) + for trial := 0; trial < 300; trial++ { + volumes := 2 + r.Intn(8) + head, tail := int64(8+r.Intn(400)), int64(7+r.Intn(60)) + set := newStoredSet(r, volumes, func(int) int { return 2 + r.Intn(7) }, head, tail, true) + bd, err := FromLocal(buildInput(SchemeFiles, nil, set.file())) + require.NoError(t, err, "trial %d", trial) + assert.Equal(t, set.truthLen, bd.Files[0].Parts, "trial %d", trial) + } +} + +// A split 7z set continues across volumes with no header: the boundary leaves +// full ranges on both sides and is only visible when the last article of a +// posted file is shorter than the rest. +func TestFromLocal_FilesSchemeSplitSevenZip(t *testing.T) { + var segs []specSeg + var truth []int + for v := 0; v < 3; v++ { + n := 4 + for j := 0; j < n; j++ { + size := int64(1000) + if j == n-1 { + size = 640 + } + s := full(fmt.Sprintf("z%d-%d@s.test", v, j), size) + if v == 0 && j == 0 { + s.start = 32 + } + if v == 2 && j == n-1 { + s.end = 400 + } + segs = append(segs, s) + } + truth = append(truth, n) + } + f := specFile{name: "movie.mkv", segs: segs} + f.size = f.fileSize() + bd, err := FromLocal(buildInput(SchemeFiles, nil, f)) + require.NoError(t, err) + assert.Equal(t, truth, bd.Files[0].Parts) +} + +// ToLocal returns the flat list with partial ranges the readers expect, equal +// field by field to the metadata the file was built from. +func TestToLocal_MultiPartFileRoundTripsFlat(t *testing.T) { + for _, scheme := range []Scheme{SchemeNZB, SchemeFiles} { + t.Run(scheme.String(), func(t *testing.T) { + r := rand.New(rand.NewSource(6)) + set := newStoredSet(r, 5, func(v int) int { return 3 + v }, 90, 30, true) + f := set.file() + var nzb [][]string + if scheme == SchemeNZB { + nzb = set.nzb() + } + in := buildInput(scheme, nzb, f) + bd, err := FromLocal(in) + require.NoError(t, err) + wire, id, err := Pack(bd) + require.NoError(t, err) + got, err := Verify(wire, id, bd.ReleaseID) + require.NoError(t, err) + loc, err := ToLocal(got, testNow) + require.NoError(t, err) + require.Len(t, loc.Files, 1) + want, have := in.Files[0].Meta, loc.Files[0].Meta + assert.Equal(t, want.FileSize, have.FileSize) + assert.Equal(t, want.OriginalFilename, have.OriginalFilename) + require.Len(t, have.SegmentData, len(want.SegmentData)) + for i := range want.SegmentData { + assert.Equal(t, want.SegmentData[i].Id, have.SegmentData[i].Id, "segment %d", i) + assert.Equal(t, want.SegmentData[i].SegmentSize, have.SegmentData[i].SegmentSize, "segment %d", i) + assert.Equal(t, want.SegmentData[i].StartOffset, have.SegmentData[i].StartOffset, "segment %d", i) + assert.Equal(t, want.SegmentData[i].EndOffset, have.SegmentData[i].EndOffset, "segment %d", i) + } + }) + } +} diff --git a/internal/peernet/bundle/fuzz_test.go b/internal/peernet/bundle/fuzz_test.go index e0ee21423..f32177221 100644 --- a/internal/peernet/bundle/fuzz_test.go +++ b/internal/peernet/bundle/fuzz_test.go @@ -8,7 +8,7 @@ import ( func seedBundles(t testing.TB) [][]byte { t.Helper() var out [][]byte - for _, b := range []*Bundle{goodBundle(t), goodNZBBundle(t), syntheticBundle(t, 50, 716_800)} { + for _, b := range []*Bundle{goodBundle(t), goodNZBBundle(t), syntheticBundle(t, 50, 716_800), partsNZBBundle(t)} { out = append(out, mustEncode(t, b)) } return out @@ -71,7 +71,7 @@ func FuzzDecompress(f *testing.F) { // mutateBundle breaks one field of a good bundle, chosen by sel. func mutateBundle(b *Bundle, sel uint8, v int64) { f := &b.Files[int(sel/32)%len(b.Files)] - switch sel % 16 { + switch sel % 18 { case 0: f.Size = v case 1: @@ -128,6 +128,12 @@ func mutateBundle(b *Bundle, sel uint8, v int64) { if len(f.Par2) > 0 && len(f.Par2[0].Segments) > 0 { f.Par2[0].Segments[0].Size = v } + case 16: + if len(f.Parts) > 0 { + f.Parts[int(sel)%len(f.Parts)] = int(v) + } + case 17: + f.Parts = append(f.Parts, int(v)) } } @@ -140,7 +146,7 @@ func FuzzValidate(f *testing.F) { f.Add([]byte{}, uint8(70), int64(576)) // The good bundles are built (and their ids hashed) once; every exec // mutates a cheap deep copy, so the fuzzer spends its time on Validate. - bases := []*Bundle{goodBundle(f), goodNZBBundle(f)} + bases := []*Bundle{goodBundle(f), goodNZBBundle(f), partsNZBBundle(f)} f.Fuzz(func(t *testing.T, data []byte, sel uint8, v int64) { check := func(b *Bundle) { if err := Validate(b); err != nil { diff --git a/internal/peernet/bundle/helpers_test.go b/internal/peernet/bundle/helpers_test.go index af32db58a..dc0aa7053 100644 --- a/internal/peernet/bundle/helpers_test.go +++ b/internal/peernet/bundle/helpers_test.go @@ -249,6 +249,7 @@ func clone(b *Bundle) *Bundle { c.Files = make([]File, len(b.Files)) for i, f := range b.Files { f.Segments = append([]Segment(nil), f.Segments...) + f.Parts = append([]int(nil), f.Parts...) f.AESKey = append([]byte(nil), f.AESKey...) f.AESIV = append([]byte(nil), f.AESIV...) f.Extents = append([]Extent(nil), f.Extents...) diff --git a/internal/peernet/bundle/limits.go b/internal/peernet/bundle/limits.go index 07c20696c..b321a3c26 100644 --- a/internal/peernet/bundle/limits.go +++ b/internal/peernet/bundle/limits.go @@ -52,6 +52,10 @@ const ( MaxTotalClips = 200_000 MaxTotalOuterSources = 50_000 MaxTotalPar2Files = 10_000 + // MaxParts caps the parts of one multi-part file (its archive volumes) and + // MaxTotalParts those of the whole bundle. + MaxParts = 10_000 + MaxTotalParts = 50_000 // MaxPar2Files caps the PAR2 file references of one file. MaxPar2Files = 1_000 diff --git a/internal/peernet/bundle/parts.go b/internal/peernet/bundle/parts.go new file mode 100644 index 000000000..2f73db1bb --- /dev/null +++ b/internal/peernet/bundle/parts.go @@ -0,0 +1,33 @@ +package bundle + +// PartsOf returns the slices of f's inline table: one per part of a multi-part +// file, or the whole table for a file read as one posted file. The slices alias +// f.Segments. A file with no inline table has none. The caller has validated f +// (the part lengths add up to the table). +func PartsOf(f *File) [][]Segment { + if len(f.Segments) == 0 { + return nil + } + if len(f.Parts) == 0 { + return [][]Segment{f.Segments} + } + out := make([][]Segment, 0, len(f.Parts)) + at := 0 + for _, n := range f.Parts { + if n < 0 || n > len(f.Segments)-at { + return [][]Segment{f.Segments} + } + out = append(out, f.Segments[at:at+n]) + at += n + } + return out +} + +// Usable is the number of bytes the usable ranges of segs add up to. +func Usable(segs []Segment) int64 { + var n int64 + for _, s := range segs { + n += s.End - s.Start + 1 + } + return n +} diff --git a/internal/peernet/bundle/parts_func_test.go b/internal/peernet/bundle/parts_func_test.go new file mode 100644 index 000000000..04ef90f2f --- /dev/null +++ b/internal/peernet/bundle/parts_func_test.go @@ -0,0 +1,20 @@ +package bundle + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPartsOf(t *testing.T) { + f := volumeFile(threeVolumes) + parts := PartsOf(&f) + if assert.Len(t, parts, 3) { + assert.Len(t, parts[0], 4) + assert.Len(t, parts[2], 3) + assert.Equal(t, f.Size, Usable(parts[0])+Usable(parts[1])+Usable(parts[2])) + } + f.Parts = nil + assert.Len(t, PartsOf(&f), 1, "no parts: the table is one posted file") + assert.Empty(t, PartsOf(&File{})) +} diff --git a/internal/peernet/bundle/parts_test.go b/internal/peernet/bundle/parts_test.go new file mode 100644 index 000000000..f6bd3d6ce --- /dev/null +++ b/internal/peernet/bundle/parts_test.go @@ -0,0 +1,416 @@ +package bundle + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" + + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +// The multi-part inline layout: a file stored in an archive set is the +// concatenation of slices, one per volume (NZB entry). These tests pin the exact +// rules; the accept-time binding to the archive headers lives in bundleimport. + +const partArticle = 1000 + +// entryBase returns the pool position of the first article of each entry. +func entryBase(entries []int) []int64 { + out := make([]int64, len(entries)) + var at int64 + for i, n := range entries { + out[i] = at + at += int64(n) + } + return out +} + +// run is a contiguous run of articles of one entry: articles [from, to] of +// entry e, the first starting at start and the last ending at end (-1: whole). +func run(entries []int, e, from, to int, start, end int64) []Segment { + base := entryBase(entries) + var out []Segment + for j := from; j <= to; j++ { + s := Segment{Ref: base[e] + int64(j), Size: partArticle, Start: 0, End: partArticle - 1} + if j == from && start > 0 { + s.Start = start + } + if j == to && end >= 0 { + s.End = end + } + out = append(out, s) + } + return out +} + +func usable(segs []Segment) int64 { + var n int64 + for _, s := range segs { + n += s.End - s.Start + 1 + } + return n +} + +func join(parts ...[]Segment) (segs []Segment, lens []int) { + for _, p := range parts { + segs = append(segs, p...) + lens = append(lens, len(p)) + } + return segs, lens +} + +// nzbBundleOf builds an nzb-scheme bundle over entries (article counts). +func nzbBundleOf(entries []int, files ...File) *Bundle { + b := &Bundle{Scheme: SchemeNZB, Title: "T", PostedUnix: 1700000000, ReleaseID: [32]byte{9}, ReleaseFiles: entries} + n := 0 + for e, c := range entries { + for j := 0; j < c; j++ { + b.Pool = append(b.Pool, fmt.Sprintf("e%d-%d@parts.test", e, j)) + n++ + } + } + for i := range files { + files[i].FileID = [32]byte{byte(i + 1)} + b.TotalSize += files[i].Size + } + b.Files = files + return b +} + +// volumeFile is a file stored across the given entries: three volumes of +// [4,4,3] articles, partial at both ends of every part as a RAR set leaves it. +func volumeFile(entries []int) File { + segs, lens := join( + run(entries, 0, 0, 3, 50, 979), + run(entries, 1, 0, 3, 40, 979), + run(entries, 2, 0, 2, 40, 499), + ) + return File{Name: "movie.mkv", Size: usable(segs), Segments: segs, Parts: lens} +} + +var threeVolumes = []int{4, 4, 3} + +func TestParts_MultiVolumeLayoutIsValid(t *testing.T) { + f := volumeFile(threeVolumes) + require.NoError(t, Validate(nzbBundleOf(threeVolumes, f))) + // The same layout inside a larger release: another entry is not imported. + require.NoError(t, Validate(nzbBundleOf([]int{4, 4, 3, 2}, volumeFile([]int{4, 4, 3, 2})))) +} + +func TestParts_RulesNZBScheme(t *testing.T) { + cases := map[string]struct { + mutate func(f *File) + want error + }{ + "interior partial range inside a part": {func(f *File) { f.Segments[1].Start = 5 }, ErrPartialRange}, + "interior short end inside a part": {func(f *File) { f.Segments[2].End = 600 }, ErrPartialRange}, + "a part spans two entries": {func(f *File) { + f.Segments[3].End, f.Segments[4].Start = partArticle-1, 0 + f.Parts, f.Size = []int{5, 3, 3}, usable(f.Segments) + }, ErrCoverage}, + "two parts of one entry in one file": {func(f *File) { + f.Segments = append(run(threeVolumes, 0, 0, 1, 50, 979), run(threeVolumes, 0, 2, 3, 0, 979)...) + f.Segments = append(f.Segments, run(threeVolumes, 2, 0, 2, 40, 499)...) + f.Parts = []int{2, 2, 3} + f.Size = usable(f.Segments) + }, ErrCoverage}, + "a part is out of article order": {func(f *File) { + f.Segments[1].Ref, f.Segments[2].Ref = f.Segments[2].Ref, f.Segments[1].Ref + }, ErrCoverage}, + "part lengths do not add up": {func(f *File) { f.Parts = []int{4, 4, 2} }, ErrInvalidField}, + "an empty part": {func(f *File) { f.Parts = []int{4, 0, 4, 3} }, ErrInvalidField}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + f := volumeFile(threeVolumes) + c.mutate(&f) + err := Validate(nzbBundleOf(threeVolumes, f)) + require.Error(t, err) + require.ErrorIs(t, err, c.want) + }) + } +} + +func TestParts_OtherNZBRules(t *testing.T) { + t.Run("a part skips an article of its entry", func(t *testing.T) { + entries := []int{5, 4, 3} + segs, lens := join( + append(run(entries, 0, 0, 2, 50, -1), run(entries, 0, 4, 4, -1, 979)...), + run(entries, 1, 0, 3, 40, 979), + run(entries, 2, 0, 2, 40, 499), + ) + f := File{Name: "movie.mkv", Size: usable(segs), Segments: segs, Parts: lens} + require.ErrorIs(t, Validate(nzbBundleOf(entries, f)), ErrCoverage) + }) + t.Run("an entry also read whole by another file", func(t *testing.T) { + whole := File{Name: "other.bin", Size: 4 * partArticle, Segments: run(threeVolumes, 1, 0, 3, -1, -1)} + require.ErrorIs(t, Validate(nzbBundleOf(threeVolumes, volumeFile(threeVolumes), whole)), ErrCoverage) + }) + t.Run("an entry also used as an outer volume", func(t *testing.T) { + nested := File{Name: "n.mkv", Size: 100, Outer: []OuterSource{{Segments: run(threeVolumes, 1, 0, 3, -1, -1)}}, + Extents: []Extent{{Outer: 0, InnerOffset: 0, InnerLength: 100}}} + require.ErrorIs(t, Validate(nzbBundleOf(threeVolumes, volumeFile(threeVolumes), nested)), ErrCoverage) + }) + t.Run("parts on a nested file", func(t *testing.T) { + nested := File{Name: "n.mkv", Size: 100, Parts: []int{1}, Outer: []OuterSource{{Segments: run(threeVolumes, 1, 0, 3, -1, -1)}}, + Extents: []Extent{{Outer: 0, InnerOffset: 0, InnerLength: 100}}} + require.ErrorIs(t, Validate(nzbBundleOf(threeVolumes, nested)), ErrInvalidField) + }) + t.Run("a single part that is a slice of the middle of an entry", func(t *testing.T) { + segs := run(threeVolumes, 1, 1, 2, 10, 20) + f := File{Name: "n.srt", Size: usable(segs), Segments: segs, Parts: []int{2}} + require.NoError(t, Validate(nzbBundleOf(threeVolumes, f))) + f.Parts = nil // without the declaration it is the original single-table layout + require.ErrorIs(t, Validate(nzbBundleOf(threeVolumes, f)), ErrCoverage) + }) +} + +// Two files stored back to back in one volume share the article where the +// first ends and the second begins, and only that. +func TestParts_SharedBoundaryArticle(t *testing.T) { + entries := []int{6} + mk := func(name string, from, to int, start, end int64) File { + segs := run(entries, 0, from, to, start, end) + return File{Name: name, Size: usable(segs), Segments: segs, Parts: []int{len(segs)}} + } + t.Run("back to back in one article", func(t *testing.T) { + a, b := mk("a.mkv", 0, 2, 30, 499), mk("b.mkv", 2, 4, 500, 699) + require.NoError(t, Validate(nzbBundleOf(entries, a, b))) + }) + t.Run("ranges overlap in the shared article", func(t *testing.T) { + a, b := mk("a.mkv", 0, 2, 30, 499), mk("b.mkv", 2, 4, 499, 699) + require.ErrorIs(t, Validate(nzbBundleOf(entries, a, b)), ErrCoverage) + }) + t.Run("shared article is interior in one of them", func(t *testing.T) { + a, b := mk("a.mkv", 0, 3, 30, 499), mk("b.mkv", 2, 4, 500, 699) + require.ErrorIs(t, Validate(nzbBundleOf(entries, a, b)), ErrCoverage) + }) + t.Run("whole article used by both", func(t *testing.T) { + a, b := mk("a.mkv", 0, 2, 30, -1), mk("b.mkv", 2, 4, 0, 699) + require.ErrorIs(t, Validate(nzbBundleOf(entries, a, b)), ErrCoverage) + }) + t.Run("disjoint article ranges of one entry", func(t *testing.T) { + a, b := mk("a.mkv", 0, 1, 30, 499), mk("b.mkv", 2, 4, 20, 699) + require.NoError(t, Validate(nzbBundleOf(entries, a, b))) + }) + t.Run("a whole-entry file may not share with a sliced one", func(t *testing.T) { + a := mk("a.mkv", 0, 2, 30, 499) + whole := File{Name: "w.bin", Size: 6 * partArticle, Segments: run(entries, 0, 0, 5, -1, -1)} + require.ErrorIs(t, Validate(nzbBundleOf(entries, a, whole)), ErrCoverage) + }) + t.Run("three tiny files in one article", func(t *testing.T) { + tiny := func(name string, start, end int64) File { + return File{Name: name, Size: end - start + 1, Segments: []Segment{{Ref: 0, Size: partArticle, Start: start, End: end}}, Parts: []int{1}} + } + require.NoError(t, Validate(nzbBundleOf([]int{1}, tiny("a.srt", 100, 199), tiny("b.srt", 200, 299), tiny("c.srt", 300, 399)))) + require.ErrorIs(t, Validate(nzbBundleOf([]int{1}, tiny("a.srt", 100, 250), tiny("b.srt", 200, 299))), ErrCoverage) + }) +} + +// The P2 shape (100 articles, each used for 10 bytes from the middle) is still +// refused as ONE table, and as parts it is only a layout that must be bound +// against an archive header later. +func TestParts_P2ShapeStaysRefused(t *testing.T) { + segs := make([]Segment, 100) + for i := range segs { + segs[i] = Segment{Ref: int64(i), Size: 700000, Start: 350000, End: 350009} + } + entries := make([]int, 100) + for i := range entries { + entries[i] = 1 + } + one := File{Name: "Movie.mkv", Size: 1000, Segments: segs} + require.ErrorIs(t, Validate(nzbBundleOf(entries, one)), ErrPartialRange) + // Interior slices of ONE part are refused too. + lens := []int{100} + require.ErrorIs(t, Validate(nzbBundleOf(entries, File{Name: "Movie.mkv", Size: 1000, Segments: segs, Parts: lens})), ErrPartialRange) +} + +func filesPartsBundle(pool int, files ...File) *Bundle { + return filesBundle(poolOf(pool), files...) +} + +func TestParts_FilesScheme(t *testing.T) { + mk := func(refs ...int64) []Segment { + var out []Segment + for i, r := range refs { + s := mkSeg(r, partArticle) + if i == 0 { + s.Start = 40 + } + if i == len(refs)-1 { + s.End = 979 + } + out = append(out, s) + } + return out + } + a, b := mk(0, 1, 2), mk(3, 4, 5) + segs := append(append([]Segment{}, a...), b...) + f := File{Name: "movie.mkv", Size: usable(segs), Segments: segs, Parts: []int{3, 3}} + require.NoError(t, Validate(filesPartsBundle(6, f))) + + t.Run("a pool entry shared by two files at their interiors", func(t *testing.T) { + other := File{Name: "other.mkv", Size: usable(mk(2, 6)), Segments: mk(2, 6), Parts: []int{2}} + require.ErrorIs(t, Validate(filesPartsBundle(7, f, other)), ErrCoverage) + }) + t.Run("an article repeated in the table", func(t *testing.T) { + bad := f + bad.Segments = append([]Segment(nil), f.Segments...) + bad.Segments[3].Ref = 0 + require.ErrorIs(t, Validate(filesPartsBundle(6, bad)), ErrCoverage) + }) + t.Run("an unreferenced pool entry is refused", func(t *testing.T) { + require.ErrorIs(t, Validate(filesPartsBundle(7, f)), ErrBadPool) + }) +} + +// partsNZBBundle is a valid, fully identified nzb-scheme bundle with one +// multi-volume file, a plain file and one NZB entry that was not imported. +func partsNZBBundle(t testing.TB) *Bundle { + t.Helper() + entries := []int{4, 4, 3, 2, 1} + vol := volumeFile(entries[:3]) + plain := File{Name: "movie.nfo", Size: 2 * partArticle, Segments: run(entries, 3, 0, 1, -1, -1)} + b := nzbBundleOf(entries, vol, plain) + b.Files = []File{vol, plain} + b.TotalSize = vol.Size + plain.Size + for i := range b.Files { + id, err := contentid.FileID(buildMeta(b, &b.Files[i])) + require.NoError(t, err) + b.Files[i].FileID = id + } + sort.Slice(b.Files, func(i, j int) bool { return bytes.Compare(b.Files[i].FileID[:], b.Files[j].FileID[:]) < 0 }) + rid, err := releaseIDOfList(b.Pool, b.ReleaseFiles) + require.NoError(t, err) + b.ReleaseID = rid + require.NoError(t, Validate(b)) + return b +} + +func TestParts_WireRoundTripKeepsPartsAndIsCanonical(t *testing.T) { + b := partsNZBBundle(t) + raw := mustEncode(t, b) + got, err := Decode(raw) + require.NoError(t, err) + require.Equal(t, b, got) + again, err := Encode(got) + require.NoError(t, err) + require.Equal(t, raw, again) + wire, id, err := Pack(b) + require.NoError(t, err) + got2, err := Verify(wire, id, b.ReleaseID) + require.NoError(t, err) + require.Equal(t, b, got2) +} + +// A bundle with no multi-part file encodes exactly as before the field existed: +// these digests were taken from the encoder before the change. +func TestParts_SinglePartBundlesKeepTheirBytes(t *testing.T) { + for name, tc := range map[string]struct { + b *Bundle + want string + }{ + "files scheme": {goodBundle(t), "53df75e95861bde57dce1593781be9e4306db25abfa6ddbd93f563ce4c70c225"}, + "nzb scheme": {goodNZBBundle(t), "cf74e63f5b2112fcc52fe57c980bb39ef77c892fe0f8673bc26faa142cfcd498"}, + } { + raw := mustEncode(t, tc.b) + sum := sha256.Sum256(raw) + assert.Equal(t, tc.want, hex.EncodeToString(sum[:]), name) + } +} + +func TestParts_DecoderIsStrict(t *testing.T) { + good := partsNZBBundle(t) + file := func() *File { c := clone(good); return fileNamed(t, c, "movie.mkv") } + d := func() *decoder { + return &decoder{segBudget: MaxSegments, extentBudget: MaxExtents, clipBudget: MaxTotalClips, + outerBudget: MaxTotalOuterSources, par2Budget: MaxTotalPar2Files, partBudget: MaxTotalParts, poolLen: int64(len(good.Pool))} + } + f := file() + enc := appendFile(f) + _, err := d().decodeFile(enc) + require.NoError(t, err) + t.Run("part lengths that do not add up", func(t *testing.T) { + g := file() + g.Parts = []int{4, 4, 2} + _, err := Decode(encodeUnchecked(replaceFile(good, "movie.mkv", g))) + require.ErrorIs(t, err, ErrInvalidField) + }) + t.Run("a zero length part", func(t *testing.T) { + g := file() + g.Parts = []int{4, 0, 4, 3} + _, err := Decode(encodeUnchecked(replaceFile(good, "movie.mkv", g))) + require.Error(t, err) + }) + t.Run("more parts than segments", func(t *testing.T) { + g := file() + g.Parts = make([]int, len(g.Segments)+1) + for i := range g.Parts { + g.Parts[i] = 1 + } + _, err := Decode(encodeUnchecked(replaceFile(good, "movie.mkv", g))) + require.Error(t, err) + }) + t.Run("an empty parts field", func(t *testing.T) { + _, err := d().decodeFile(append(append([]byte{}, enc...), appendBytes(nil, fParts, nil)...)) + require.ErrorIs(t, err, ErrMalformed) + }) + t.Run("the parts field twice", func(t *testing.T) { + _, err := d().decodeFile(append(append([]byte{}, enc...), enc[len(enc)-len(appendParts(f.Parts)):]...)) + require.ErrorIs(t, err, ErrMalformed) + }) + t.Run("the parts field before the file id", func(t *testing.T) { + tail := appendParts(f.Parts) + _, err := d().decodeFile(append(append(append([]byte{}, enc[:len(enc)-len(tail)-34]...), tail...), enc[len(enc)-len(tail)-34:len(enc)-len(tail)]...)) + require.ErrorIs(t, err, ErrMalformed) + }) + t.Run("a non minimal varint is not the canonical encoding", func(t *testing.T) { + tail := appendParts(f.Parts) + raw := mustEncode(t, good) + // 4 -> 0x84 0x00 (same value, longer encoding), then fix the lengths by re-wrapping. + pb := []byte{0x84, 0x00} + for _, n := range f.Parts[1:] { + pb = protowire.AppendVarint(pb, uint64(n)) + } + g := appendBytes(enc[:len(enc)-len(tail)], fParts, pb) + raw2 := rebuildWithFile(t, raw, good, "movie.mkv", g) + _, err := Decode(raw2) + require.ErrorIs(t, err, ErrMalformed) + }) +} + +func replaceFile(b *Bundle, name string, f *File) *Bundle { + c := clone(b) + for i := range c.Files { + if c.Files[i].Name == name { + c.Files[i] = *f + } + } + return c +} + +// rebuildWithFile re-wraps the top-level fields of raw with the named file's +// bytes replaced by fileBytes. +func rebuildWithFile(t testing.TB, raw []byte, b *Bundle, name string, fileBytes []byte) []byte { + t.Helper() + var out []rawField + idx := 0 + for _, f := range splitTop(t, raw) { + if f.num == fFiles { + if b.Files[idx].Name == name { + f.b = fileBytes + } + idx++ + } + out = append(out, f) + } + return joinTop(out) +} diff --git a/internal/peernet/bundle/types.go b/internal/peernet/bundle/types.go index bb65d48c0..b5da18630 100644 --- a/internal/peernet/bundle/types.go +++ b/internal/peernet/bundle/types.go @@ -91,6 +91,11 @@ type File struct { FileID contentid.ID // Segments is the file's own segment table (empty for nested files). Segments []Segment + // Parts, when set, makes the table a multi-part inline layout: the table is + // the concatenation of slices, one per part, and Parts lists how many + // segments each part holds. Empty means the table is read as one posted + // file (the original layout; it encodes without this field). + Parts []int // AESKey and AESIV are set together for a directly AES-encrypted file. AESKey []byte AESIV []byte diff --git a/internal/peernet/bundle/validate.go b/internal/peernet/bundle/validate.go index 072b1c896..e4fa65cd8 100644 --- a/internal/peernet/bundle/validate.go +++ b/internal/peernet/bundle/validate.go @@ -83,10 +83,27 @@ type validator struct { entLen []int inline map[int]struct{} sharedE map[int]struct{} + // slicedE are the entries read by the parts of a multi-part file. + slicedE map[int]struct{} + // uses lists, per pool entry, the usable ranges of the parts that use it at + // a part boundary (see checkSlicedTable). + uses map[int64][]articleUse + parts int total int64 lastFile *File } +// articleUse is the usable range one part takes from a boundary article. +type articleUse struct{ start, end int64 } + +// Owner marks of a pool entry used by the parts of a multi-part file: it is +// shared at a part boundary (and listed in validator.uses), or used inside a +// part, which no other part may touch. +const ( + ownerBoundary int32 = -2 + ownerInterior int32 = -3 +) + // Validate checks every structural rule of the bundle schema: limits, ranges, // sums, encryption sizes, names and the message-ID pool. It does not touch the // network or recompute identifiers (Verify does that). It never panics and @@ -127,7 +144,7 @@ func Validate(b *Bundle) error { } if b.Scheme == SchemeFiles { for i, o := range v.owner { - if o < 0 { + if o == -1 { return fmt.Errorf("%w: pool entry %d is not referenced", ErrBadPool, i) } } @@ -187,7 +204,7 @@ func (v *validator) checkPool() error { v.entStart[i] = pos pos += n } - v.inline, v.sharedE = map[int]struct{}{}, map[int]struct{}{} + v.inline, v.sharedE, v.slicedE = map[int]struct{}{}, map[int]struct{}{}, map[int]struct{}{} v.initOwner() return nil } @@ -251,6 +268,8 @@ func (v *validator) checkFile(f *File) error { nested := len(f.Extents) > 0 || len(f.Outer) > 0 switch { + case len(f.Parts) > 0 && (nested || len(f.Segments) == 0): + return fmt.Errorf("%w: parts on a file without an inline table", ErrInvalidField) case nested && len(f.Segments) > 0: return fmt.Errorf("%w: nested file lists inline segments", ErrInvalidField) case nested && len(f.AESKey) > 0: @@ -322,7 +341,13 @@ func (v *validator) checkInline(f *File) error { if len(f.Segments) == 0 { return fmt.Errorf("%w: file has no data", ErrInvalidField) } - covered, err := v.checkTable(f.Segments) + var covered int64 + var err error + if len(f.Parts) > 0 { + covered, err = v.checkSlicedTable(f) + } else { + covered, err = v.checkTable(f.Segments) + } if err != nil { return err } @@ -361,6 +386,33 @@ func KeyOf(segs []Segment) TableKey { return k } +// checkSegment validates one segment of a table (or of one part of a table). +// first and last say whether it is the first or last segment of the run that +// may be read partially at that end. +func (v *validator) checkSegment(i int, s Segment, first, last bool) error { + poolLen := int64(len(v.b.Pool)) + ceiling := int64(MaxSegmentSize) + if v.b.Scheme == SchemeFiles { + ceiling = MaxFilesSchemeSegment + } + switch { + case s.Ref == GapRef: + // A shareable file has no known holes: a gap is never accepted. + return fmt.Errorf("%w: segment %d is a gap", ErrInvalidField, i) + case s.Ref < 0 || s.Ref >= poolLen: + return fmt.Errorf("%w: segment %d references pool entry %d of %d", ErrOutOfRange, i, s.Ref, poolLen) + case s.Size > ceiling: + return fmt.Errorf("%w: segment %d is %d bytes", ErrImplausible, i, s.Size) + case s.Size < 1: + return fmt.Errorf("%w: segment %d size %d", ErrOutOfRange, i, s.Size) + case s.Start < 0 || s.End < s.Start || s.End > s.Size-1: + return fmt.Errorf("%w: segment %d range [%d,%d] of %d", ErrOutOfRange, i, s.Start, s.End, s.Size) + case !first && s.Start != 0, !last && s.End != s.Size-1: + return fmt.Errorf("%w: segment %d range [%d,%d] of %d", ErrPartialRange, i, s.Start, s.End, s.Size) + } + return nil +} + // checkTable validates one segment table and returns the bytes it covers. // // A table reads one posted file front to back. Only its first segment may @@ -372,28 +424,11 @@ func (v *validator) checkTable(segs []Segment) (int64, error) { if v.segs += len(segs); v.segs > MaxSegments { return 0, fmt.Errorf("%w: more than %d segments", ErrLimit, MaxSegments) } - poolLen := int64(len(v.b.Pool)) - ceiling := int64(MaxSegmentSize) - if v.b.Scheme == SchemeFiles { - ceiling = MaxFilesSchemeSegment - } seen := make(map[int64]struct{}, len(segs)) var covered int64 for i, s := range segs { - switch { - case s.Ref == GapRef: - // A shareable file has no known holes: a gap is never accepted. - return 0, fmt.Errorf("%w: segment %d is a gap", ErrInvalidField, i) - case s.Ref < 0 || s.Ref >= poolLen: - return 0, fmt.Errorf("%w: segment %d references pool entry %d of %d", ErrOutOfRange, i, s.Ref, poolLen) - case s.Size > ceiling: - return 0, fmt.Errorf("%w: segment %d is %d bytes", ErrImplausible, i, s.Size) - case s.Size < 1: - return 0, fmt.Errorf("%w: segment %d size %d", ErrOutOfRange, i, s.Size) - case s.Start < 0 || s.End < s.Start || s.End > s.Size-1: - return 0, fmt.Errorf("%w: segment %d range [%d,%d] of %d", ErrOutOfRange, i, s.Start, s.End, s.Size) - case i > 0 && s.Start != 0, i < len(segs)-1 && s.End != s.Size-1: - return 0, fmt.Errorf("%w: segment %d range [%d,%d] of %d", ErrPartialRange, i, s.Start, s.End, s.Size) + if err := v.checkSegment(i, s, i == 0, i == len(segs)-1); err != nil { + return 0, err } if _, dup := seen[s.Ref]; dup { return 0, fmt.Errorf("%w: segment %d repeats pool entry %d within one table", ErrCoverage, i, s.Ref) @@ -408,7 +443,7 @@ func (v *validator) checkTable(segs []Segment) (int64, error) { id := int32(len(v.tables)) v.tables[key] = id for i, s := range segs { - if v.owner[s.Ref] >= 0 { + if v.owner[s.Ref] != -1 { return 0, fmt.Errorf("%w: segment %d shares pool entry %d with a different table", ErrCoverage, i, s.Ref) } v.owner[s.Ref] = id @@ -416,6 +451,88 @@ func (v *validator) checkTable(segs []Segment) (int64, error) { return covered, nil } +// checkSlicedTable validates the table of a multi-part file: the concatenation +// of Parts, each a run of one posted file read from some byte of its first +// article to some byte of its last. Inside a part the rules of a table hold +// (only the part's first segment may start inside its article, only its last +// may end inside it); a part boundary is the only place a usable range may skip +// bytes. A pool entry is used once in the whole table. Two parts of different +// files may share an article only at their boundaries (a part's first or last +// segment) with disjoint usable ranges: two files stored back to back in one +// archive volume share the article where one ends and the next begins. Any +// other use of an entry by two tables is refused. +func (v *validator) checkSlicedTable(f *File) (int64, error) { + segs := f.Segments + if len(f.Parts) > MaxParts { + return 0, fmt.Errorf("%w: %d parts", ErrLimit, len(f.Parts)) + } + if v.parts += len(f.Parts); v.parts > MaxTotalParts { + return 0, fmt.Errorf("%w: more than %d parts in the bundle", ErrLimit, MaxTotalParts) + } + sum := 0 + for i, n := range f.Parts { + if n < 1 || n > len(segs)-sum { + return 0, fmt.Errorf("%w: part %d holds %d segments", ErrInvalidField, i, n) + } + sum += n + } + if sum != len(segs) { + return 0, fmt.Errorf("%w: parts hold %d of %d segments", ErrInvalidField, sum, len(segs)) + } + if v.segs += len(segs); v.segs > MaxSegments { + return 0, fmt.Errorf("%w: more than %d segments", ErrLimit, MaxSegments) + } + if v.uses == nil { + v.uses = map[int64][]articleUse{} + } + seen := make(map[int64]struct{}, len(segs)) + var covered int64 + at := 0 + for _, n := range f.Parts { + for k := 0; k < n; k++ { + i, s := at+k, segs[at+k] + first, last := k == 0, k == n-1 + if err := v.checkSegment(i, s, first, last); err != nil { + return 0, err + } + if _, dup := seen[s.Ref]; dup { + return 0, fmt.Errorf("%w: segment %d repeats pool entry %d within one table", ErrCoverage, i, s.Ref) + } + seen[s.Ref] = struct{}{} + if err := v.shareArticle(i, s, first || last); err != nil { + return 0, err + } + covered += s.End - s.Start + 1 + } + at += n + } + return covered, nil +} + +// shareArticle records the use of a pool entry by a segment of a part and +// refuses every use the boundary exception does not cover. +func (v *validator) shareArticle(i int, s Segment, boundary bool) error { + use := articleUse{start: s.Start, end: s.End} + switch o := v.owner[s.Ref]; { + case o == -1: + if !boundary { + v.owner[s.Ref] = ownerInterior + return nil + } + v.owner[s.Ref] = ownerBoundary + case o == ownerBoundary && boundary: + for _, u := range v.uses[s.Ref] { + if use.start <= u.end && u.start <= use.end { + return fmt.Errorf("%w: segment %d overlaps the range another part uses of pool entry %d", ErrCoverage, i, s.Ref) + } + } + default: + return fmt.Errorf("%w: segment %d shares pool entry %d with another table, not at a part boundary", ErrCoverage, i, s.Ref) + } + v.uses[s.Ref] = append(v.uses[s.Ref], use) + return nil +} + func (v *validator) checkNested(f *File) error { if len(f.Extents) == 0 { return fmt.Errorf("%w: outer sources without extents", ErrInvalidField) @@ -604,6 +721,9 @@ func (v *validator) checkCoverage(f *File) error { return fmt.Errorf("%w: NZB entry %d is read twice by one file", ErrCoverage, e) } own[e] = struct{}{} + if _, sliced := v.slicedE[e]; sliced { + return fmt.Errorf("%w: NZB entry %d is used by a multi-part file and elsewhere", ErrCoverage, e) + } _, wasInline := v.inline[e] if wasInline || (inline && v.shared(e)) { return fmt.Errorf("%w: NZB entry %d is used inline and elsewhere", ErrCoverage, e) @@ -615,7 +735,12 @@ func (v *validator) checkCoverage(f *File) error { } return nil } - if len(f.Segments) > 0 { + switch { + case len(f.Parts) > 0: + if err := v.slicedCoverage(f, own); err != nil { + return err + } + case len(f.Segments) > 0: if err := use(f.Segments, true); err != nil { return err } @@ -636,3 +761,39 @@ func (v *validator) checkCoverage(f *File) error { func (v *validator) shared(e int) bool { _, ok := v.sharedE[e]; return ok } func (v *validator) markShared(e int) { v.sharedE[e] = struct{}{} } + +// slicedCoverage (nzb scheme) requires every part of a multi-part file to be a +// contiguous run, in article order, of ONE NZB entry, and every entry to supply +// at most one part to the file. Entries are not required to be in NZB order (an +// obfuscated NZB lists a volume set in any order); the order of the parts is the +// order of the file's bytes, which the accept-time binding checks against the +// archive's own headers. An entry used by a multi-part file is never also read +// whole, as an outer volume or as a PAR2 file. +func (v *validator) slicedCoverage(f *File, own map[int]struct{}) error { + at := 0 + for pi, n := range f.Parts { + part := f.Segments[at : at+n] + at += n + e := v.entryOf(part[0].Ref) + if e >= len(v.entStart) { + return fmt.Errorf("%w: part %d starts outside the release list", ErrCoverage, pi) + } + for i, s := range part { + if s.Ref != part[0].Ref+int64(i) { + return fmt.Errorf("%w: part %d segment %d is out of article order or skips an article", ErrCoverage, pi, i) + } + } + if end := int64(v.entStart[e] + v.entLen[e]); part[len(part)-1].Ref >= end { + return fmt.Errorf("%w: part %d spans more than one NZB file entry", ErrCoverage, pi) + } + if _, dup := own[e]; dup { + return fmt.Errorf("%w: NZB entry %d supplies two parts of one file", ErrCoverage, e) + } + own[e] = struct{}{} + if _, wasInline := v.inline[e]; wasInline || v.shared(e) { + return fmt.Errorf("%w: NZB entry %d is used by a multi-part file and elsewhere", ErrCoverage, e) + } + v.slicedE[e] = struct{}{} + } + return nil +} diff --git a/internal/peernet/bundle/wire.go b/internal/peernet/bundle/wire.go index ee2b4889f..6bbafa0e0 100644 --- a/internal/peernet/bundle/wire.go +++ b/internal/peernet/bundle/wire.go @@ -35,6 +35,7 @@ const ( fClips protowire.Number = 9 fPar2 protowire.Number = 10 fFileID protowire.Number = 11 + fParts protowire.Number = 12 // OuterSource. fOuterSegs protowire.Number = 1 @@ -132,6 +133,19 @@ func sameRun(a, b Segment) bool { return b.Ref == a.Ref+1 } +// appendParts encodes the part lengths of a multi-part file as one packed +// field. It is empty for a file read as one posted file. +func appendParts(parts []int) []byte { + if len(parts) == 0 { + return nil + } + var pb []byte + for _, n := range parts { + pb = protowire.AppendVarint(pb, uint64(n)) + } + return appendBytes(nil, fParts, pb) +} + func appendFile(f *File) []byte { var b []byte b = appendString(b, fName, f.Name) @@ -182,7 +196,8 @@ func appendFile(f *File) []byte { pb = appendBytes(pb, fPar2Segs, appendTable(p.Segments)) b = appendBytes(b, fPar2, pb) } - return appendBytes(b, fFileID, f.FileID[:]) + b = appendBytes(b, fFileID, f.FileID[:]) + return append(b, appendParts(f.Parts)...) } // encodeUnchecked writes the canonical bytes without validating b. Callers diff --git a/internal/peernet/bundle/wire_decode.go b/internal/peernet/bundle/wire_decode.go index 5d1b70e3d..9c2d58c8f 100644 --- a/internal/peernet/bundle/wire_decode.go +++ b/internal/peernet/bundle/wire_decode.go @@ -124,6 +124,7 @@ type decoder struct { clipBudget int outerBudget int par2Budget int + partBudget int poolLen int64 } @@ -422,6 +423,8 @@ func (d *decoder) decodeFile(b []byte) (File, error) { case fFileID: err = fixedField(fl, f.FileID[:], "file id") haveID = true + case fParts: + f.Parts, err = d.decodeParts(fl, len(f.Segments)) default: err = fmt.Errorf("%w: unknown file field %d", ErrMalformed, fl.num) } @@ -467,6 +470,40 @@ func (d *decoder) decodeExtents(f field) ([]Extent, error) { return out, nil } +// decodeParts reads the part lengths of a multi-part file. The field is only +// ever written for a file that has parts, so an empty one is not canonical. The +// count is bounded by the segments already decoded and by the bundle-wide part +// budget before anything is allocated. +func (d *decoder) decodeParts(f field, segments int) ([]int, error) { + b, err := bytesField(f, "parts") + if err != nil { + return nil, err + } + nv := countVarints(b) + if nv == 0 { + return nil, fmt.Errorf("%w: empty parts field", ErrMalformed) + } + if nv > MaxParts || nv > segments { + return nil, fmt.Errorf("%w: %d parts for %d segments", ErrLimit, nv, segments) + } + if err := take(&d.partBudget, nv, "parts", MaxTotalParts); err != nil { + return nil, err + } + out := make([]int, 0, nv) + r := varReader{b: b} + for r.more() { + n, err := r.next() + if err != nil { + return nil, err + } + if n > MaxSegments { + return nil, fmt.Errorf("%w: part length", ErrInvalidField) + } + out = append(out, int(n)) + } + return out, nil +} + func (d *decoder) decodeClips(f field) ([]Clip, error) { b, err := bytesField(f, "clips") if err != nil { @@ -548,7 +585,7 @@ func decodeRaw(raw []byte) (*Bundle, error) { } b := &Bundle{} d := &decoder{segBudget: MaxSegments, extentBudget: MaxExtents, clipBudget: MaxTotalClips, - outerBudget: MaxTotalOuterSources, par2Budget: MaxTotalPar2Files} + outerBudget: MaxTotalOuterSources, par2Budget: MaxTotalPar2Files, partBudget: MaxTotalParts} var val *validator r := fieldReader{b: raw, repeated: isRepeatedBundleField} have := 0 diff --git a/internal/peernet/bundleimport/accept.go b/internal/peernet/bundleimport/accept.go index 3129740f9..be9099700 100644 --- a/internal/peernet/bundleimport/accept.go +++ b/internal/peernet/bundleimport/accept.go @@ -148,6 +148,8 @@ func (i *Importer) Accept(ctx context.Context, req Request) (*Result, error) { } return i.rejected(r, err) } + res.Cost = Cost{Articles: rep.articles, Bytes: rep.bytes, ExtentsBound: rep.bound, ExtentsUnbound: rep.checked - rep.bound, + PartsBound: rep.partsBound, PartsUnbound: rep.partsUnbound} if r, err := i.probeMedia(ctx, loc, stage, needsStructure(rep, b)); err != nil { if r == ReasonMediaInvalid || r == ReasonStructureInvalid { i.noteFailedBundle(ctx, req.Origin.BundleID, b.ReleaseID.String()) diff --git a/internal/peernet/bundleimport/cost.go b/internal/peernet/bundleimport/cost.go index 467502f52..0d487ca6a 100644 --- a/internal/peernet/bundleimport/cost.go +++ b/internal/peernet/bundleimport/cost.go @@ -34,6 +34,18 @@ func (c *costBudget) charge(n int64) error { return nil } +// chargeMany takes bytes for n articles at once (the mandatory samples). +func (c *costBudget) chargeMany(bytes int64, n int) error { + c.mu.Lock() + defer c.mu.Unlock() + if bytes < 0 || c.used+bytes > c.limit { + return errCost + } + c.used += bytes + c.n += n + return nil +} + // left is what remains. func (c *costBudget) left() int64 { c.mu.Lock() diff --git a/internal/peernet/bundleimport/extents.go b/internal/peernet/bundleimport/extents.go index 03c6fb87e..9b9de24bb 100644 --- a/internal/peernet/bundleimport/extents.go +++ b/internal/peernet/bundleimport/extents.go @@ -210,6 +210,11 @@ type extentReport struct { unbound map[int]string bound int checked int + // partsBound and partsUnbound count the header checks of multi-part files. + partsBound, partsUnbound int + // articles and bytes are what the shared budget was charged. + articles int + bytes int64 } // verifyExtents runs the header check for the picked extents. A mismatch is diff --git a/internal/peernet/bundleimport/parts.go b/internal/peernet/bundleimport/parts.go new file mode 100644 index 000000000..1402b5218 --- /dev/null +++ b/internal/peernet/bundleimport/parts.go @@ -0,0 +1,366 @@ +package bundleimport + +import ( + "context" + "fmt" + "sort" + "sync" + + "github.com/kipsilabs/altmount/internal/importer/parser/fileinfo" + "github.com/kipsilabs/altmount/internal/peernet/archivehdr" + "github.com/kipsilabs/altmount/internal/peernet/bundle" +) + +// Multi-part inline layouts. +// +// A file stored in an archive set (a video in part01.rar ... partNN.rar) is the +// concatenation of one slice per volume: from the byte after the volume's own +// headers to the byte before its trailer. The bundle declares the slices +// (File.Parts); this file is what the receiver does with them. +// +// Yenc. A part that is a whole NZB entry (nzb scheme), or whose ranges are full +// (files scheme), claims to be a whole posted file and is checked as one: its +// first article begins at 0 and the posted file's total is the sum of the part. +// Any other part is a slice of a larger posted file: every sampled article must +// agree on where the part begins in it and on its total (relState). The last +// article of every part is always sampled, so a size lie anywhere in a part +// shows as a disagreement between the end and the other samples. +// +// Archive headers. A part that is a slice is bound to the archive's own file +// header: the data area of a stored file entry starts at the byte where the part +// starts, is as long as the part, and its continuation flags fit the part's place +// in the file; for RAR 5 the volume numbers must run in the order of the parts. +// A part that claims to be a whole media file is checked not to be an archive +// volume. A part that cannot be judged (a slice that does not begin at its posted +// file's first article, an unknown or encoded archive, a spent budget) is +// unverifiable, and the file is then accepted only when its media structure check +// passes and its container has a real structural check. + +// maxPartChecks bounds how many parts of a bundle get an archive header check. +const maxPartChecks = 64 + +// entryIndex answers which NZB entry a pool position belongs to (nzb scheme). +type entryIndex struct { + start []int64 + n []int +} + +func newEntryIndex(b *bundle.Bundle) *entryIndex { + if b.Scheme != bundle.SchemeNZB { + return &entryIndex{} + } + e := &entryIndex{start: make([]int64, len(b.ReleaseFiles)), n: b.ReleaseFiles} + var at int64 + for i, n := range b.ReleaseFiles { + e.start[i] = at + at += int64(n) + } + return e +} + +// whole reports whether segs are exactly the articles of one NZB entry. +func (e *entryIndex) whole(segs []bundle.Segment) bool { + if len(segs) == 0 || len(e.start) == 0 { + return false + } + k := sort.Search(len(e.start), func(i int) bool { return e.start[i]+int64(e.n[i]) > segs[0].Ref }) + for k < len(e.start) && e.n[k] == 0 { + k++ + } + return k < len(e.start) && segs[0].Ref == e.start[k] && len(segs) == e.n[k] +} + +func fullRange(segs []bundle.Segment) bool { + return segs[0].Start == 0 && segs[len(segs)-1].End == segs[len(segs)-1].Size-1 +} + +// wholeYenc says the part is checked as a whole posted file: it is a whole NZB +// entry, or (files scheme, no entries) its ends are full. +func (e *entryIndex) wholeYenc(segs []bundle.Segment) bool { + if len(e.start) > 0 { + return e.whole(segs) + } + return fullRange(segs) +} + +// partsOfSliced returns the parts of a multi-part file (nil for any other). +func partsOfSliced(f *bundle.File) [][]bundle.Segment { + if len(f.Parts) == 0 { + return nil + } + return bundle.PartsOf(f) +} + +// partKind is what the header check does for one part. +type partKind uint8 + +const ( + // partSlice: bind the part to a stored file entry of its volume. + partSlice partKind = iota + // partRaw: the part claims to be a whole posted file of media data; it must + // not start like an archive volume. + partRaw +) + +type partRef struct { + file, part int + kind partKind +} + +// classifyPart says which header check a part needs, and whether it needs one. +func classifyPart(ents *entryIndex, f *bundle.File, ps []bundle.Segment) (partKind, bool) { + whole := ents.wholeYenc(ps) + switch { + case whole && fullRange(ps): + return partRaw, fileinfo.IsVerifiableMediaFile(f.Name) + default: + return partSlice, true + } +} + +// pickParts chooses the parts to check: all of them up to maxPartChecks, then +// the first and last part of every media file, then a uniform random choice +// (crypto/rand) of the rest. Slices come before raw parts. +func pickParts(b *bundle.Bundle) []partRef { + ents := newEntryIndex(b) + var all []partRef + for fi := range b.Files { + f := &b.Files[fi] + for k, ps := range partsOfSliced(f) { + if kind, ok := classifyPart(ents, f, ps); ok { + all = append(all, partRef{fi, k, kind}) + } + } + } + if len(all) <= maxPartChecks { + return all + } + var picks, rest []partRef + for _, r := range all { + f := &b.Files[r.file] + if fileinfo.IsVerifiableMediaFile(f.Name) && (r.part == 0 || r.part == len(f.Parts)-1) && len(picks) < maxPartChecks/2 { + picks = append(picks, r) + } else { + rest = append(rest, r) + } + } + shuffle(rest) + return append(picks, rest[:maxPartChecks-len(picks)]...) +} + +// partReserve is the cost the header checks of the picked parts are expected to +// have: a few articles each. +func partReserve(b *bundle.Bundle, picks []partRef) int64 { + var n int64 + for _, r := range picks { + first := partsOfSliced(&b.Files[r.file])[r.part][0] + n += extentReserveArticles * first.Size + } + return n +} + +// markNeed0 makes the first article of every picked part a mandatory sample. +func markNeed0(tables []sampleTable, picks []partRef) { + at := map[[2]int]int{} + for i := range tables { + if tables[i].part { + at[[2]int{tables[i].file, tables[i].index}] = i + } + } + for _, r := range picks { + if i, ok := at[[2]int{r.file, r.part}]; ok { + tables[i].need0 = true + } + } +} + +type partWork struct { + ref partRef + segs []bundle.Segment + // raw is the posted volume as far as the part reaches, read from its first + // byte; offset is where the part starts in it. + raw *bundle.OuterSource + offset int64 + name string + upTo int64 + vol *archivehdr.Volume + err error + skip string + table int +} + +// rawView is the part's articles read whole from the first byte of the first +// article, as the posted volume begins. +func rawView(ps []bundle.Segment) *bundle.OuterSource { + raw := &bundle.OuterSource{Segments: make([]bundle.Segment, len(ps))} + for i, s := range ps { + raw.Segments[i] = bundle.Segment{Ref: s.Ref, Size: s.Size, Start: 0, End: s.Size - 1} + } + return raw +} + +// verifyParts runs the header checks of the picked parts. +func (i *Importer) verifyParts(ctx context.Context, b *bundle.Bundle, tables []sampleTable, st Staging, picks []partRef, rel *relState, bud *costBudget, rep *extentReport) (Reason, bool, error) { + if len(picks) == 0 { + return "", false, nil + } + byTable := map[[2]int]int{} + for k := range tables { + if tables[k].part { + byTable[[2]int{tables[k].file, tables[k].index}] = k + } + } + var work []*partWork + ents := newEntryIndex(b) + for n, r := range picks { + f := &b.Files[r.file] + ps := partsOfSliced(f)[r.part] + w := &partWork{ref: r, segs: ps, raw: rawView(ps), name: fmt.Sprintf("part-%d.bin", n), upTo: ps[0].Start, offset: ps[0].Start, table: byTable[[2]int{r.file, r.part}]} + if r.kind == partRaw { + w.upTo = 0 + } else { + i.placePart(ctx, b, ents, w, rel, bud) + } + if w.skip == "" { + if err := st.WriteFileMetadata(w.name, outerMeta(b, w.raw)); err != nil { + return ReasonStaging, false, err + } + } + work = append(work, w) + } + ctx, cancel := context.WithTimeout(ctx, i.cfg.Policy.YencTimeoutOrDefault()) + defer cancel() + var wg sync.WaitGroup + sem := make(chan struct{}, 4) + for _, w := range work { + if w.skip != "" { + continue + } + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + w.vol, w.err = scanVolume(ctx, st, &volumeWork{name: w.name, outer: w.raw, upTo: w.upTo}, bud) + }() + } + wg.Wait() + volumes := map[int][]*partWork{} + for _, w := range work { + f := &b.Files[w.ref.file] + if w.err != nil { + if reason, hard := readFailure(w.err); hard { + return reason, false, fmt.Errorf("%s (part %d): %w", f.Name, w.ref.part+1, w.err) + } + } + res := checkPart(w, f, len(f.Parts)) + switch res.Outcome { + case archivehdr.Bound: + rep.partsBound++ + if w.vol != nil { + volumes[w.ref.file] = append(volumes[w.ref.file], w) + } + case archivehdr.Mismatch: + return ReasonExtentMismatch, true, fmt.Errorf("%s: part %d of %d: %s", f.Name, w.ref.part+1, len(f.Parts), res.Detail) + default: + rep.partsUnbound++ + if _, ok := rep.unbound[w.ref.file]; !ok { + rep.unbound[w.ref.file] = res.Detail + } + } + } + for fi, ws := range volumes { + if d := volumeOrder(ws); d != "" { + return ReasonExtentMismatch, true, fmt.Errorf("%s: %s", b.Files[fi].Name, d) + } + } + return "", false, nil +} + +// placePart decides what the header walk of a slice reads. A part that begins at +// the first article of its posted file is read from there. In the nzb scheme a +// part that begins further into its volume (a second file stored after the +// first) is read through a view that starts at the volume's first article: the +// articles before the part are taken to be as long as the first article +// (a poster's articles are, but the last), which the part's own offset has to +// confirm (it must equal the number of articles before it times that length). +// Anything else cannot be judged and is left unverifiable. +func (i *Importer) placePart(ctx context.Context, b *bundle.Bundle, ents *entryIndex, w *partWork, rel *relState, bud *costBudget) { + first, ok := rel.first[w.table] + switch { + case !ok: + w.skip = "the first article of the part was not read" + return + case first == 0: + return + case len(ents.start) == 0: + w.skip = "the part does not begin at the first article of its posted file" + return + } + k := sort.Search(len(ents.start), func(k int) bool { return ents.start[k]+int64(ents.n[k]) > w.segs[0].Ref }) + if k >= len(ents.start) { + w.skip = "the part is not inside an NZB entry" + return + } + before := w.segs[0].Ref - ents.start[k] + h, err := i.cfg.Headers.SegmentHeader(ctx, b.Pool[ents.start[k]]) + switch { + case err != nil: + w.skip = "the first article of the posted file could not be read: " + err.Error() + return + case h.PartBegin != 0 || h.PartSize < 1 || before*h.PartSize != first: + w.skip = "the articles before the part are not as long as the first one" + return + case bud.charge(h.PartSize) != nil: + w.skip = "the verification budget is spent" + return + } + view := &bundle.OuterSource{} + for j := int64(0); j < before; j++ { + view.Segments = append(view.Segments, bundle.Segment{Ref: ents.start[k] + j, Size: h.PartSize, Start: 0, End: h.PartSize - 1}) + } + view.Segments = append(view.Segments, w.raw.Segments...) + w.raw, w.offset, w.upTo = view, first+w.segs[0].Start, first+w.segs[0].Start +} + +// checkPart judges one scanned part. +func checkPart(w *partWork, f *bundle.File, count int) archivehdr.Result { + if w.skip != "" { + return archivehdr.Result{Outcome: archivehdr.Unverifiable, Detail: w.skip} + } + if w.ref.kind == partRaw { + if w.vol != nil && w.vol.Format != archivehdr.FormatUnknown { + return archivehdr.Result{Outcome: archivehdr.Mismatch, + Detail: fmt.Sprintf("the part claims to be whole media data but begins like a %s archive volume", w.vol.Format)} + } + return archivehdr.Result{Outcome: archivehdr.Bound} + } + e := archivehdr.Extent{Offset: w.offset, Length: bundle.Usable(w.segs), Index: w.ref.part, Count: count, FileSize: f.Size, Keyed: len(f.AESKey) > 0} + if e.Keyed && w.ref.part == 0 { + e.IV = f.AESIV + } + return archivehdr.Check(w.vol, w.err, e) +} + +// volumeOrder requires the RAR 5 volume numbers of the checked parts of one file +// to run in the order of the parts (consecutive volumes). A volume that declares +// no number (RAR 4, 7z) says nothing. +func volumeOrder(ws []*partWork) string { + type seen struct { + part int + vol int64 + } + var known []seen + for _, w := range ws { + if w.vol != nil && w.vol.VolumeNumber >= 0 { + known = append(known, seen{w.ref.part, w.vol.VolumeNumber}) + } + } + for k := 1; k < len(known); k++ { + if d := known[k].vol - known[0].vol; d != int64(known[k].part-known[0].part) { + return fmt.Sprintf("part %d is volume %d and part %d is volume %d: the volumes are not in the order of the parts", + known[0].part+1, known[0].vol+1, known[k].part+1, known[k].vol+1) + } + } + return "" +} diff --git a/internal/peernet/bundleimport/parts_test.go b/internal/peernet/bundleimport/parts_test.go new file mode 100644 index 000000000..a14698cbc --- /dev/null +++ b/internal/peernet/bundleimport/parts_test.go @@ -0,0 +1,822 @@ +package bundleimport + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/archivehdr" + "github.com/kipsilabs/altmount/internal/peernet/archivehdr/archivetest" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +// A file stored in a multi-volume archive set is read as the concatenation of +// slices, one per volume (the multi-part inline layout). These tests build real +// stored archive volumes, post them through the fake pool as yEnc articles and +// run the whole accept pipeline over bundles that describe them. + +type setSpec struct { + tag string + kind string // rar4, rar5 or 7z + payload []byte + // cuts is how many payload bytes each volume holds (rar4, rar5) or the + // volume sizes of one split 7z archive (7z). + cuts []int + segSize int + scheme bundle.Scheme + // extra, when set, is stored as extra.nfo after the movie's data in volume + // extraVol (RAR only). + extra []byte + extraVol int +} + +type builtSet struct { + b *bundle.Bundle + posted [][]byte + // lo and hi are the movie's usable range inside every posted volume. + lo, hi []int + ids [][]string + names []string + spec setSpec + // extraLo and extraHi are extra.nfo's data range inside volume spec.extraVol. + extraLo, extraHi int +} + +func volumeName(tag string, v int) string { return fmt.Sprintf("%s.part%02d.rar", tag, v+1) } + +type artSlice struct { + art, size, start, end int +} + +// slices lists the articles of one posted volume that [lo, hi) touches, with +// the usable range inside each. +func volSlices(posted []byte, segSize, lo, hi int) []artSlice { + var out []artSlice + for i := 0; i*segSize < len(posted); i++ { + from, end := i*segSize, min((i+1)*segSize, len(posted)) + if end > lo && from < hi { + out = append(out, artSlice{i, end - from, max(lo-from, 0), min(hi, end) - from - 1}) + } + } + return out +} + +func (e *env) mkStoredSet(sp setSpec) builtSet { + e.t.Helper() + if sp.segSize == 0 { + sp.segSize = 128 + } + s := builtSet{spec: sp} + total := len(sp.payload) + var extraLo, extraHi int + switch sp.kind { + case "7z": + a := archivetest.SevenZip(archivetest.SevenZipOpts{}, archivetest.File{Name: "movie.mkv", Data: sp.payload}) + at := 0 + for _, n := range sp.cuts { + s.posted = append(s.posted, a.Data[at:at+n]) + s.lo = append(s.lo, int(max(a.DataOffsets[0]-int64(at), 0))) + s.hi = append(s.hi, int(min(a.DataOffsets[0]+int64(total)-int64(at), int64(n)))) + at += n + } + require.Equal(e.t, len(a.Data), at, "cuts must cover the archive") + default: + at := 0 + for v, n := range sp.cuts { + files := []archivetest.File{{Name: "movie.mkv", Data: sp.payload[at : at+n], SplitBefore: v > 0, SplitAfter: v < len(sp.cuts)-1, UnpackedSize: int64(total)}} + if v == sp.extraVol && sp.extra != nil { + files = append(files, archivetest.File{Name: "extra.nfo", Data: sp.extra}) + } + var a archivetest.Built + if sp.kind == "rar5" { + a = archivetest.RAR5Volume(v, files...) + } else { + a = archivetest.RAR4(files...) + } + s.posted = append(s.posted, a.Data) + s.lo = append(s.lo, int(a.DataOffsets[0])) + s.hi = append(s.hi, int(a.DataOffsets[0])+n) + if v == sp.extraVol && sp.extra != nil { + extraLo, extraHi = int(a.DataOffsets[1]), int(a.DataOffsets[1])+len(sp.extra) + } + at += n + } + } + for v, posted := range s.posted { + var ids []string + for i := 0; i*sp.segSize < len(posted); i++ { + from, end := i*sp.segSize, min((i+1)*sp.segSize, len(posted)) + id := fmt.Sprintf("%s-v%d-%03d@test", sp.tag, v, i) + beh := fakepoolBehavior(posted[from:end], from, len(posted)) + beh.YEnc.FileName = volumeName(sp.tag, v) + e.fp.SetBehavior(id, beh) + ids = append(ids, id) + } + s.ids = append(s.ids, ids) + s.names = append(s.names, volumeName(sp.tag, v)) + } + // The pool: the whole NZB list (nzb scheme) or the referenced articles in + // first-use order (files scheme). + b := &bundle.Bundle{Scheme: sp.scheme, Title: "Test Release " + sp.tag, PostedUnix: 1_700_000_000} + used := map[string]bool{} + movie := bundle.File{Name: "movie.mkv", Size: int64(total)} + extra := bundle.File{Name: "extra.nfo", Size: int64(len(sp.extra))} + type pending struct { + file *bundle.File + id string + sl artSlice + firstOfNew bool + } + var order []pending + for v, posted := range s.posted { + first := true + for _, sl := range volSlices(posted, sp.segSize, s.lo[v], s.hi[v]) { + order = append(order, pending{&movie, s.ids[v][sl.art], sl, first}) + first = false + used[s.ids[v][sl.art]] = true + } + movie.Parts = append(movie.Parts, len(volSlices(posted, sp.segSize, s.lo[v], s.hi[v]))) + if v == sp.extraVol && sp.extra != nil { + n := 0 + for _, sl := range volSlices(posted, sp.segSize, extraLo, extraHi) { + order = append(order, pending{&extra, s.ids[v][sl.art], sl, false}) + used[s.ids[v][sl.art]] = true + n++ + } + extra.Parts = []int{n} + } + } + if sp.scheme == bundle.SchemeNZB { + for _, ids := range s.ids { + b.ReleaseFiles = append(b.ReleaseFiles, len(ids)) + b.Pool = append(b.Pool, ids...) + } + } else { + for _, p := range order { + if !contains(b.Pool, p.id) { + b.Pool = append(b.Pool, p.id) + } + } + } + for _, p := range order { + p.file.Segments = append(p.file.Segments, bundle.Segment{Ref: poolPos(b, p.id), Size: int64(p.sl.size), Start: int64(p.sl.start), End: int64(p.sl.end)}) + } + b.Files = []bundle.File{movie} + if sp.extra != nil { + b.Files = append(b.Files, extra) + } + for _, f := range b.Files { + b.TotalSize += f.Size + } + s.b = b + s.extraLo, s.extraHi = extraLo, extraHi + e.sealAny(b) + return s +} + +func contains(pool []string, id string) bool { return poolPos(&bundle.Bundle{Pool: pool}, id) >= 0 } + +func poolPos(b *bundle.Bundle, id string) int64 { + for i, p := range b.Pool { + if p == id { + return int64(i) + } + } + return -1 +} + +func (e *env) nzbReleaseID(b *bundle.Bundle) contentid.ID { + store := &metapb.NzbStore{} + pos := 0 + for _, n := range b.ReleaseFiles { + fe := &metapb.NzbFileEntry{} + for _, id := range b.Pool[pos : pos+n] { + fe.Segments = append(fe.Segments, &metapb.NzbSeg{Id: id}) + } + pos += n + store.Files = append(store.Files, fe) + } + rid, err := contentid.ReleaseID(store) + require.NoError(e.t, err) + return rid +} + +// sealAny recomputes the identifiers of any scheme after a test changed a bundle. +func (e *env) sealAny(b *bundle.Bundle) { + e.t.Helper() + require.NoError(e.t, e.tryReseal(b)) +} + +func (e *env) tryReseal(b *bundle.Bundle) error { + for i := range b.Files { + b.Files[i].FileID = contentid.ID{byte(i + 1)} + } + b.ReleaseID = contentid.ID{9} + loc, err := bundle.ToLocal(b, e.imp.cfg.Now()) + if err != nil { + return err + } + ids := make([]contentid.ID, len(b.Files)) + for i := range b.Files { + id, err := contentid.FileID(loc.Files[i].Meta) + if err != nil { + return err + } + b.Files[i].FileID = id + } + sort.Slice(b.Files, func(i, j int) bool { return string(b.Files[i].FileID[:]) < string(b.Files[j].FileID[:]) }) + for i := range b.Files { + ids[i] = b.Files[i].FileID + } + if b.Scheme == bundle.SchemeFiles { + b.ReleaseID = bundle.FilesReleaseID(ids) + } else { + b.ReleaseID = e.nzbReleaseID(b) + } + return bundle.Validate(b) +} + +func (s builtSet) movie() *bundle.File { + for i := range s.b.Files { + if s.b.Files[i].Name == "movie.mkv" { + return &s.b.Files[i] + } + } + return nil +} + +// fastPath makes the request carry the user's NZB the way the importer's fast +// path does: the segment list, encoded sizes and subject file names. +func (s builtSet) fastPath(req Request) Request { + req.ExpectedNZB = s.ids + req.ExpectedNZBNames = s.names + for v, posted := range s.posted { + var sizes []int64 + for i := range s.ids[v] { + sizes = append(sizes, int64(min(s.spec.segSize, len(posted)-i*s.spec.segSize))+4) + } + req.ExpectedNZBSizes = append(req.ExpectedNZBSizes, sizes) + } + return req +} + +func (e *env) stagedBytes(path string) []byte { + e.t.Helper() + meta, err := e.lib.ReadFileMetadata(path) + require.NoError(e.t, err) + st, err := e.imp.cfg.Stager.NewStaging() + require.NoError(e.t, err) + defer func() { _ = st.Close() }() + require.NoError(e.t, st.WriteFileMetadata("read.bin", meta)) + f, err := st.Opener().OpenFile(context.Background(), "read.bin", os.O_RDONLY, 0) + require.NoError(e.t, err) + defer func() { _ = f.Close() }() + out, err := io.ReadAll(f) + require.NoError(e.t, err) + return out +} + +func evenCuts(total, n int) []int { + out := make([]int, n) + for i := range out { + out[i] = total / n + } + out[n-1] += total % n + return out +} + +// (a) and (b): honest sets, every format and scheme, open and private networks. +func TestParts_HonestStoredSetsAreAcceptedAndReadBackIdentically(t *testing.T) { + for _, kind := range []string{"rar5", "rar4"} { + for _, scheme := range []bundle.Scheme{bundle.SchemeNZB, bundle.SchemeFiles} { + for _, volumes := range []int{3, 5} { + for _, open := range []bool{false, true} { + name := fmt.Sprintf("%s/%s/%d volumes/open=%v", kind, scheme, volumes, open) + t.Run(name, func(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(1500) + s := e.mkStoredSet(setSpec{tag: "h", kind: kind, payload: payload, cuts: evenCuts(len(payload), volumes), scheme: scheme, extraVol: -1}) + req := e.req(s.b) + req.Origin.Open = open + if open { + req.Origin.BundleID = strings.Repeat("e", 64) + } + res, err := e.imp.Accept(context.Background(), req) + require.NoError(t, err, "%v", err) + require.Len(t, res.Paths, 1) + assert.Equal(t, payload, e.stagedBytes(res.Paths[0]), "the staged file reads the same bytes as the original") + assert.Empty(t, e.rep.got) + assert.Equal(t, volumes, res.Cost.PartsBound, "every part is bound to its archive header") + assert.Zero(t, res.Cost.PartsUnbound) + }) + } + } + } + } +} + +// (b) The importer's fast path: the user's NZB is present, so the Release ID, the +// declared sizes and the subject names of every entry are checked too. +func TestParts_FastPathChecksEntriesNamesAndSizes(t *testing.T) { + for _, kind := range []string{"rar5", "rar4"} { + t.Run(kind, func(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(1500) + s := e.mkStoredSet(setSpec{tag: "fp", kind: kind, payload: payload, cuts: evenCuts(len(payload), 4), scheme: bundle.SchemeNZB, extraVol: -1}) + res, err := e.imp.Accept(context.Background(), s.fastPath(e.openReq(s.b))) + require.NoError(t, err, "%v", err) + assert.Equal(t, payload, e.stagedBytes(res.Paths[0])) + }) + } + t.Run("an entry named differently by the user's NZB", func(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(1500) + s := e.mkStoredSet(setSpec{tag: "fn", kind: "rar5", payload: payload, cuts: evenCuts(len(payload), 4), scheme: bundle.SchemeNZB, extraVol: -1}) + req := s.fastPath(e.openReq(s.b)) + req.ExpectedNZBNames[2] = "something.else.rar" + _, err := e.imp.Accept(context.Background(), req) + require.Error(t, err) + assert.Equal(t, ReasonYencMismatch, ReasonOf(err), "%v", err) + assert.Empty(t, e.rep.got, "a disagreement with the local NZB is not the publisher's fault") + e.assertNothingBehind() + }) + t.Run("a segment list that is not the user's NZB", func(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(1500) + s := e.mkStoredSet(setSpec{tag: "fl", kind: "rar5", payload: payload, cuts: evenCuts(len(payload), 4), scheme: bundle.SchemeNZB, extraVol: -1}) + req := s.fastPath(e.openReq(s.b)) + req.ExpectedNZB = req.ExpectedNZB[:3] + _, err := e.imp.Accept(context.Background(), req) + assert.Equal(t, ReasonNZBMismatch, ReasonOf(err)) + }) +} + +// (d) A split 7z archive continues across volumes with no header of its own: the +// middle volumes are whole posted files, the first and last are slices whose +// header (at the end of the last volume) cannot be read, so the file is +// accepted on its media structure alone. +func TestParts_SplitSevenZipIsAcceptedOnItsStructure(t *testing.T) { + for _, scheme := range []bundle.Scheme{bundle.SchemeNZB, bundle.SchemeFiles} { + t.Run(scheme.String(), func(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(1500) + total := len(archivetest.SevenZip(archivetest.SevenZipOpts{}, archivetest.File{Name: "movie.mkv", Data: payload}).Data) + cuts := evenCuts(total, 4) + s := e.mkStoredSet(setSpec{tag: "z", kind: "7z", payload: payload, cuts: cuts, scheme: scheme, extraVol: -1}) + res, err := e.imp.Accept(context.Background(), e.openReq(s.b)) + require.NoError(t, err, "%v", err) + assert.Equal(t, payload, e.stagedBytes(res.Paths[0])) + assert.Positive(t, res.Cost.PartsUnbound, "the 7z end header is not in the volume that holds the slice") + }) + } +} + +// (c) Hostile layouts over a real set. +func TestParts_HostileLayoutsAreRejectedWithTheRightReason(t *testing.T) { + type hostile struct { + kinds []string + volumes int + mutate func(t *testing.T, e *env, s *builtSet) + reason Reason + report bool + // detail is what the rejection must say, so each case is caught by the + // check it is about. + detail string + } + setSeg := func(s *builtSet, part int, f func(first, last *bundle.Segment)) { + m := s.movie() + at := 0 + for k := 0; k < part; k++ { + at += m.Parts[k] + } + f(&m.Segments[at], &m.Segments[at+m.Parts[part]-1]) + m.Size = 0 + for _, g := range m.Segments { + m.Size += g.End - g.Start + 1 + } + s.b.TotalSize = m.Size + } + swapParts := func(s *builtSet, a, b int) { + m := s.movie() + var chunks [][]bundle.Segment + at := 0 + for _, n := range m.Parts { + chunks = append(chunks, m.Segments[at:at+n]) + at += n + } + chunks[a], chunks[b] = chunks[b], chunks[a] + m.Parts[a], m.Parts[b] = m.Parts[b], m.Parts[a] + var flat []bundle.Segment + for _, c := range chunks { + flat = append(flat, c...) + } + m.Segments = flat + } + cases := map[string]hostile{ + "a part shifted 6 bytes into the header area": {[]string{"rar5", "rar4"}, 3, func(t *testing.T, e *env, s *builtSet) { + setSeg(s, 1, func(first, last *bundle.Segment) { first.Start -= 6; last.End -= 6 }) + }, ReasonExtentMismatch, true, "part 2 of 3: "}, + "a part shifted 5 bytes into the data": {[]string{"rar5", "rar4"}, 3, func(t *testing.T, e *env, s *builtSet) { + setSeg(s, 1, func(first, last *bundle.Segment) { first.Start += 5; last.End += 5 }) + }, ReasonExtentMismatch, true, "part 2 of 3: "}, + "a part one byte too long, the file size kept": {[]string{"rar5", "rar4"}, 3, func(t *testing.T, e *env, s *builtSet) { + setSeg(s, 0, func(_, last *bundle.Segment) { last.End++ }) + setSeg(s, 2, func(_, last *bundle.Segment) { last.End-- }) + }, ReasonExtentMismatch, true, "part 1 of 3: the header says 300 bytes in this volume, the layout 301"}, + "a file size that is not the archive's": {[]string{"rar5", "rar4"}, 3, func(t *testing.T, e *env, s *builtSet) { + setSeg(s, 2, func(_, last *bundle.Segment) { last.End-- }) + }, ReasonExtentMismatch, true, "the header says the file is 900 bytes"}, + "first and last part swapped": {[]string{"rar5", "rar4"}, 4, func(t *testing.T, e *env, s *builtSet) { swapParts(s, 0, 3) }, + ReasonExtentMismatch, true, "continuation flag"}, + "two middle parts swapped": {[]string{"rar5"}, 5, func(t *testing.T, e *env, s *builtSet) { swapParts(s, 1, 2) }, + ReasonExtentMismatch, true, "not in the order of the parts"}, + } + for label, c := range cases { + for _, kind := range c.kinds { + for _, scheme := range []bundle.Scheme{bundle.SchemeNZB, bundle.SchemeFiles} { + t.Run(fmt.Sprintf("%s/%s/%s", label, kind, scheme), func(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(300 * c.volumes) + s := e.mkStoredSet(setSpec{tag: "x", kind: kind, payload: payload, cuts: evenCuts(len(payload), c.volumes), scheme: scheme, extraVol: -1}) + c.mutate(t, e, &s) + require.NoError(t, e.tryReseal(s.b), "the mutated layout must still be structurally valid") + _, err := e.imp.Accept(context.Background(), e.openReq(s.b)) + require.Error(t, err) + assert.Equal(t, c.reason, ReasonOf(err), "%v", err) + assert.Contains(t, err.Error(), c.detail) + if c.report { + assert.Len(t, e.rep.got, 1, "the publisher is reported") + assert.Len(t, e.rep.failedBundles, 1, "the Bundle ID goes on the negative cache") + } + e.assertNothingBehind() + }) + } + } + } +} + +// Two files stored back to back in one volume: the movie's last slice and the +// subtitle share the article where one ends and the next begins. +func TestParts_SecondFileInTheLastVolumeIsAcceptedToo(t *testing.T) { + for _, kind := range []string{"rar5", "rar4"} { + for _, scheme := range []bundle.Scheme{bundle.SchemeNZB, bundle.SchemeFiles} { + t.Run(kind+"/"+scheme.String(), func(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(900) + nfo := []byte(strings.Repeat("a release note\n", 20)) + s := e.mkStoredSet(setSpec{tag: "sf", kind: kind, payload: payload, cuts: evenCuts(len(payload), 3), scheme: scheme, extra: nfo, extraVol: 2}) + res, err := e.imp.Accept(context.Background(), e.openReq(s.b)) + require.NoError(t, err, "%v", err) + assert.Len(t, res.Paths, 2) + for _, p := range res.Paths { + if strings.HasSuffix(p, "movie.mkv") { + assert.Equal(t, payload, e.stagedBytes(p)) + } + } + }) + } + } +} + +func (s builtSet) dropExtra() { + var keep []bundle.File + for _, f := range s.b.Files { + if f.Name == "movie.mkv" { + keep = append(keep, f) + } + } + s.b.Files = keep + s.b.TotalSize = keep[0].Size +} + +// A part may not point at another stored file of the same volume: the entry +// that starts where the part starts must be the file's own continuation. +func TestParts_PartPointingAtAnotherStoredFileIsRejected(t *testing.T) { + for _, kind := range []string{"rar5", "rar4"} { + for _, scheme := range []bundle.Scheme{bundle.SchemeNZB, bundle.SchemeFiles} { + t.Run(kind+"/"+scheme.String(), func(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(900) + nfo := []byte(strings.Repeat("N", 300)) // as long as a part of the movie + s := e.mkStoredSet(setSpec{tag: "ot", kind: kind, payload: payload, cuts: evenCuts(len(payload), 3), scheme: scheme, extra: nfo, extraVol: 1}) + s.dropExtra() + m := s.movie() + // part 1 (volume 1) now reads extra.nfo's data area instead of the movie's slice + vol := s.posted[1] + var segs []bundle.Segment + for _, sl := range volSlices(vol, s.spec.segSize, s.extraLo, s.extraHi) { + segs = append(segs, bundle.Segment{Ref: poolPos(s.b, s.ids[1][sl.art]), Size: int64(sl.size), Start: int64(sl.start), End: int64(sl.end)}) + } + at := m.Parts[0] + m.Segments = append(append(append([]bundle.Segment{}, m.Segments[:at]...), segs...), m.Segments[at+m.Parts[1]:]...) + m.Parts[1] = len(segs) + if scheme == bundle.SchemeFiles { + require.NoError(t, dropUnusedPool(s.b)) + } + require.NoError(t, e.tryReseal(s.b)) + res, err := e.imp.Accept(context.Background(), e.openReq(s.b)) + if scheme == bundle.SchemeFiles { + // Without the NZB list the articles before the part are unknown, so + // a slice that does not begin at its posted file's first article is + // unverifiable: it is accepted only on the media structure, and + // reported as unbound. + if err == nil { + assert.Positive(t, res.Cost.PartsUnbound) + } + return + } + require.Error(t, err) + assert.Equal(t, ReasonExtentMismatch, ReasonOf(err), "%v", err) + assert.Contains(t, err.Error(), "part 2 of 3") + assert.Len(t, e.rep.got, 1) + e.assertNothingBehind() + }) + } + } +} + +// A part that claims to be a whole posted file (full ranges) of media data may +// not be an archive volume, and a files-scheme part with full ranges must be a +// whole posted file. +func TestParts_PartDeclaredWholeThatIsNotIsRejected(t *testing.T) { + fullPart := func(s *builtSet, part int) { + m := s.movie() + at := 0 + for k := 0; k < part; k++ { + at += m.Parts[k] + } + for k := at; k < at+m.Parts[part]; k++ { + m.Segments[k].Start, m.Segments[k].End = 0, m.Segments[k].Size-1 + } + m.Size = bundle.Usable(m.Segments) + s.b.TotalSize = m.Size + } + for _, kind := range []string{"rar5", "rar4"} { + for _, scheme := range []bundle.Scheme{bundle.SchemeNZB, bundle.SchemeFiles} { + t.Run("a whole archive volume/"+kind+"/"+scheme.String(), func(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(900) + s := e.mkStoredSet(setSpec{tag: "wh", kind: kind, payload: payload, cuts: evenCuts(len(payload), 3), scheme: scheme, extraVol: -1}) + fullPart(&s, 0) + require.NoError(t, e.tryReseal(s.b)) + _, err := e.imp.Accept(context.Background(), e.openReq(s.b)) + require.Error(t, err) + assert.Equal(t, ReasonExtentMismatch, ReasonOf(err), "%v", err) + assert.Contains(t, err.Error(), "begins like a") + e.assertNothingBehind() + }) + } + t.Run("full ranges taken from the middle of a posted file/"+kind, func(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(900) + s := e.mkStoredSet(setSpec{tag: "mid", kind: kind, payload: payload, cuts: evenCuts(len(payload), 3), scheme: bundle.SchemeFiles, extraVol: -1}) + m := s.movie() + // part 1 keeps only its interior articles, full: not a whole posted file + at := m.Parts[0] + n := m.Parts[1] + inner := append([]bundle.Segment{}, m.Segments[at+1:at+n-1]...) + for k := range inner { + inner[k].Start, inner[k].End = 0, inner[k].Size-1 + } + m.Segments = append(append(append([]bundle.Segment{}, m.Segments[:at]...), inner...), m.Segments[at+n:]...) + m.Parts[1] = len(inner) + m.Size = bundle.Usable(m.Segments) + s.b.TotalSize = m.Size + // the dropped articles are no longer referenced: the files scheme needs a pool of used ones + require.NoError(t, dropUnusedPool(s.b)) + require.NoError(t, e.tryReseal(s.b)) + _, err := e.imp.Accept(context.Background(), e.openReq(s.b)) + require.Error(t, err) + assert.Equal(t, ReasonYencMismatch, ReasonOf(err), "%v", err) + assert.Len(t, e.rep.got, 1, "attributable") + }) + } +} + +func dropUnusedPool(b *bundle.Bundle) error { + used := map[int64]bool{} + for _, f := range b.Files { + for _, s := range f.Segments { + used[s.Ref] = true + } + } + remap := map[int64]int64{} + var pool []string + for i, id := range b.Pool { + if used[int64(i)] { + remap[int64(i)] = int64(len(pool)) + pool = append(pool, id) + } + } + b.Pool = pool + for i := range b.Files { + for k := range b.Files[i].Segments { + b.Files[i].Segments[k].Ref = remap[b.Files[i].Segments[k].Ref] + } + } + return nil +} + +// Layouts the bundle's own rules refuse never reach the network checks. +func TestParts_StructurallyInvalidLayoutsAreRefusedBeforeAnyRead(t *testing.T) { + t.Run("an interior partial range", func(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(900) + s := e.mkStoredSet(setSpec{tag: "ip", kind: "rar5", payload: payload, cuts: evenCuts(len(payload), 3), scheme: bundle.SchemeNZB, extraVol: -1}) + m := s.movie() + m.Segments[1].Start = 3 + _, err := e.imp.Accept(context.Background(), e.openReq(s.b)) + assert.Equal(t, ReasonInvalid, ReasonOf(err), "%v", err) + assert.ErrorIs(t, bundle.Validate(s.b), bundle.ErrPartialRange) + }) + t.Run("two files claiming overlapping ranges of one volume", func(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(900) + nfo := []byte(strings.Repeat("N", 300)) + s := e.mkStoredSet(setSpec{tag: "ov", kind: "rar5", payload: payload, cuts: evenCuts(len(payload), 3), scheme: bundle.SchemeNZB, extra: nfo, extraVol: 2}) + var nfoFile, movie *bundle.File + for i := range s.b.Files { + if s.b.Files[i].Name == "extra.nfo" { + nfoFile = &s.b.Files[i] + } else { + movie = &s.b.Files[i] + } + } + // the subtitle claims the movie's last bytes of that volume + last := movie.Segments[len(movie.Segments)-1] + nfoFile.Segments = []bundle.Segment{{Ref: last.Ref, Size: last.Size, Start: last.End - 9, End: last.End}} + nfoFile.Size, nfoFile.Parts = 10, []int{1} + s.b.TotalSize = movie.Size + 10 + _, err := e.imp.Accept(context.Background(), e.openReq(s.b)) + assert.Equal(t, ReasonInvalid, ReasonOf(err), "%v", err) + assert.ErrorIs(t, bundle.Validate(s.b), bundle.ErrCoverage) + }) +} + +// The P2 shape: 100 volumes, each contributing a 10-byte slice from the middle of +// its article. No stored file's header binds such slices, so it is not accepted +// as a multi-part layout either. +func TestParts_P2ShapeIsRejectedAsAMultiPartLayout(t *testing.T) { + for _, scheme := range []bundle.Scheme{bundle.SchemeNZB, bundle.SchemeFiles} { + t.Run(scheme.String(), func(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(100 * 300) + s := e.mkStoredSet(setSpec{tag: "p2", kind: "rar5", payload: payload, cuts: evenCuts(len(payload), 100), segSize: 1024, scheme: scheme, extraVol: -1}) + m := s.movie() + require.Len(t, m.Parts, 100) + for k := range m.Segments { + m.Segments[k].Start = int64(s.lo[k] + 100) + m.Segments[k].End = m.Segments[k].Start + 9 + } + m.Size = 1000 + s.b.TotalSize = 1000 + require.NoError(t, e.tryReseal(s.b)) + _, err := e.imp.Accept(context.Background(), e.openReq(s.b)) + require.Error(t, err) + assert.Equal(t, ReasonExtentMismatch, ReasonOf(err), "%v", err) + e.assertNothingBehind() + }) + } +} + +// What an honest stored set costs to verify. +func TestParts_VerificationCostByVolumeCount(t *testing.T) { + for _, n := range []int{50, 200} { + t.Run(fmt.Sprintf("%d volumes", n), func(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(n * 200) + s := e.mkStoredSet(setSpec{tag: "c", kind: "rar5", payload: payload, cuts: evenCuts(len(payload), n), segSize: 1024, scheme: bundle.SchemeNZB, extraVol: -1}) + res, err := e.imp.Accept(context.Background(), e.openReq(s.b)) + require.NoError(t, err, "%v", err) + checked := min(n, maxPartChecks) + t.Logf("%d volumes (one article each): %d articles read, %d header checks (%d bound)", n, res.Cost.Articles, checked, res.Cost.PartsBound) + assert.Equal(t, checked, res.Cost.PartsBound) + // one article per part (it is first and last at once) plus the header read + // of every checked part + assert.Equal(t, n+checked, res.Cost.Articles) + }) + } +} + +// The forced full check samples headers of the stored (flat) metadata, which +// no longer says where the parts are; the volume boundaries are recovered from +// the ranges, so an honest multi-volume file does not fail it. +func TestVerifyPeerHeaders_MultiVolumeFileIsNotAFalseAlarm(t *testing.T) { + for _, kind := range []string{"rar5", "rar4"} { + t.Run(kind, func(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(1500) + s := e.mkStoredSet(setSpec{tag: "vh", kind: kind, payload: payload, cuts: evenCuts(len(payload), 4), scheme: bundle.SchemeNZB, extraVol: -1}) + res, err := e.imp.Accept(context.Background(), e.openReq(s.b)) + require.NoError(t, err) + require.NoError(t, e.imp.VerifyPeerHeaders(context.Background(), res.Paths[0])) + }) + } +} + +func TestVerifyPeerHeaders_MultiVolumeLieInAnInteriorArticleIsFound(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(1500) + s := e.mkStoredSet(setSpec{tag: "vl", kind: "rar5", payload: payload, cuts: evenCuts(len(payload), 4), scheme: bundle.SchemeNZB, extraVol: -1}) + res, err := e.imp.Accept(context.Background(), e.openReq(s.b)) + require.NoError(t, err) + id := s.ids[2][2] + beh := e.fp.BehaviorFor(id) + beh.YEnc.PartBegin += 5 + e.fp.SetBehavior(id, beh) + var lm *LayoutMismatchError + require.ErrorAs(t, e.imp.VerifyPeerHeaders(context.Background(), res.Paths[0]), &lm) +} + +// Volumes written by the real archiver (the importer's fixtures): the header +// checks bind every slice, and the staged file reads the original bytes. +func TestParts_RealArchiverVolumesAreBoundAndReadBack(t *testing.T) { + root := filepath.Join("..", "..", "importer", "testdata") + cases := []struct{ dir, glob string }{{"rar_multi", "archive.part*.rar"}, {"rar_widthmismatch", "archive.part*.rar"}} + for _, tc := range cases { + t.Run(tc.dir, func(t *testing.T) { + matches, err := filepath.Glob(filepath.Join(root, tc.dir, tc.glob)) + require.NoError(t, err) + require.NotEmpty(t, matches) + // natural order: part02 before part010 + sort.Slice(matches, func(a, b int) bool { + if len(matches[a]) != len(matches[b]) { + return len(matches[a]) < len(matches[b]) + } + return matches[a] < matches[b] + }) + payload, err := os.ReadFile(filepath.Join(root, "payload_a.bin")) + require.NoError(t, err) + + e := newEnv(t) + const segSize = 8192 + var nzb [][]string + var segs []*metapb.SegmentData + var names []string + for v, m := range matches { + data, err := os.ReadFile(m) + require.NoError(t, err) + names = append(names, filepath.Base(m)) + vol, err := archivehdr.Scan(bytes.NewReader(data), int64(len(data)), int64(len(data)), archivehdr.Limits{MaxBlocks: 64, MaxReads: 400}) + require.NoError(t, err) + assert.EqualValues(t, v, vol.VolumeNumber, "the real archiver's volume numbers run in volume order") + var blk *archivehdr.Block + for k := range vol.Blocks { + if vol.Blocks[k].File { + blk = &vol.Blocks[k] + } + } + require.NotNil(t, blk) + var ids []string + for i := 0; i*segSize < len(data); i++ { + from, end := i*segSize, min((i+1)*segSize, len(data)) + id := fmt.Sprintf("real-%s-v%d-%03d@test", tc.dir, v, i) + beh := fakepoolBehavior(data[from:end], from, len(data)) + beh.YEnc.FileName = filepath.Base(m) + e.fp.SetBehavior(id, beh) + ids = append(ids, id) + } + nzb = append(nzb, ids) + for _, sl := range volSlices(data, segSize, int(blk.DataOffset), int(blk.DataOffset+blk.DataSize)) { + segs = append(segs, &metapb.SegmentData{Id: ids[sl.art], SegmentSize: int64(sl.size), StartOffset: int64(sl.start), EndOffset: int64(sl.end)}) + } + } + in := bundle.ReleaseInput{Scheme: bundle.SchemeNZB, Title: "Real Archive", PostedAt: time.Unix(1_700_000_000, 0), NZBFiles: nzb, + Files: []bundle.FileInput{{Name: "movie.mkv", Meta: &metapb.FileMetadata{FileSize: int64(len(payload)), SegmentData: segs, Status: metapb.FileStatus_FILE_STATUS_HEALTHY}}}} + bd, err := bundle.FromLocal(in) + require.NoError(t, err) + require.Len(t, bd.Files[0].Parts, len(matches)) + + stage, closeStage := e.imp.stager() + defer closeStage() + rep, _, _, err := e.imp.verifyYenc(context.Background(), bd, names, stage) + require.NoError(t, err) + assert.Equal(t, len(matches), rep.partsBound, "every volume's slice is bound to its own file header") + assert.Zero(t, rep.partsUnbound) + + loc, err := bundle.ToLocal(bd, time.Unix(1_700_000_100, 0)) + require.NoError(t, err) + st, _, err := stage() + require.NoError(t, err) + require.NoError(t, st.WriteFileMetadata("read.bin", loc.Files[0].Meta)) + f, err := st.Opener().OpenFile(context.Background(), "read.bin", os.O_RDONLY, 0) + require.NoError(t, err) + defer func() { _ = f.Close() }() + got, err := io.ReadAll(f) + require.NoError(t, err) + assert.Equal(t, payload, got) + }) + } +} diff --git a/internal/peernet/bundleimport/sample.go b/internal/peernet/bundleimport/sample.go index 6640e961d..c2ebc7dbf 100644 --- a/internal/peernet/bundleimport/sample.go +++ b/internal/peernet/bundleimport/sample.go @@ -67,6 +67,20 @@ type sampleTable struct { // nzbName is the file name the user's NZB subject gives the NZB entry this // table reads ("" when unknown); the first article's yEnc name must agree. nzbName string + // part is set for a table that is one part of a multi-part file. A part + // that is not a whole posted file (rel) is checked relative to where its + // first article begins in that file, which the sampled headers reveal; a + // whole one is checked as before (offset 0, file total = sum of the part). + part bool + rel bool + // mainPart marks a part of the main media file: its interior samples are + // planned over all the parts together (planMust). + mainPart bool + // file and index place a part table in its bundle file. + file, index int + // need0 makes the first article a mandatory sample (the header binding of + // the part needs where it begins). + need0 bool } // sample is one article to read. A mandatory one must be answered for the @@ -99,7 +113,7 @@ func entryNameAt(b *bundle.Bundle, names []string, pos int64) string { func tablesOf(b *bundle.Bundle, names []string) []sampleTable { var out []sampleTable seen := map[bundle.TableKey]struct{}{} - add := func(name string, segs []bundle.Segment, main bool) { + add := func(name string, segs []bundle.Segment, main bool, tag func(*sampleTable)) { if len(segs) == 0 { return } @@ -114,14 +128,26 @@ func tablesOf(b *bundle.Bundle, names []string) []sampleTable { t.ids[i], t.sizes[i], t.prefix[i] = b.Pool[s.Ref], s.Size, t.total t.total += s.Size } + if tag != nil { + tag(&t) + } out = append(out, t) } + ents := newEntryIndex(b) mainFile := mainMediaFile(b) for i := range b.Files { f := &b.Files[i] - add(f.Name, f.Segments, i == mainFile) + if len(f.Parts) == 0 { + add(f.Name, f.Segments, i == mainFile, nil) + } + for k, ps := range partsOfSliced(f) { + add(fmt.Sprintf("%s (part %d of %d)", f.Name, k+1, len(f.Parts)), ps, false, func(t *sampleTable) { + t.part, t.file, t.index, t.mainPart = true, i, k, i == mainFile + t.rel = !ents.wholeYenc(ps) + }) + } for j := range f.Outer { - add(fmt.Sprintf("%s (volume %d)", f.Name, j+1), f.Outer[j].Segments, i == mainFile) + add(fmt.Sprintf("%s (volume %d)", f.Name, j+1), f.Outer[j].Segments, i == mainFile, nil) } } return out @@ -216,6 +242,9 @@ func planMust(tables []sampleTable) (plan []sample, bytes int64) { continue } push(t, n-1) + if tables[t].need0 { + push(t, 0) + } if !tables[t].main { continue } @@ -227,9 +256,43 @@ func planMust(tables []sampleTable) (plan []sample, bytes int64) { } } } + planMainParts(tables, push) return plan, bytes } +// planMainParts plans the interior samples of the parts of the main media file as +// if they were one table: one per mediaPerSegments segments of the whole file, at +// least three, drawn uniformly over all their interior articles. +func planMainParts(tables []sampleTable, push func(t, pos int) bool) { + var at []int // table of every interior position, flattened + var pos []int + for t := range tables { + if !tables[t].mainPart { + continue + } + for p := 1; p < len(tables[t].ids)-1; p++ { + at, pos = append(at, t), append(pos, p) + } + } + want := min(max(3, (len(at)+2*countParts(tables))/mediaPerSegments), len(at)) + for k, tries := 0, 0; k < want && tries < 32*want+32; tries++ { + g := randN(len(at)) + if push(at[g], pos[g]) { + k++ + } + } +} + +func countParts(tables []sampleTable) int { + n := 0 + for i := range tables { + if tables[i].mainPart { + n++ + } + } + return n +} + // planOptional spends what is left of the budget on the first article of // randomly chosen tables and then on one random interior article of randomly // chosen tables. taken lists the samples already planned. @@ -317,12 +380,15 @@ func SubjectFilename(subject string) string { // The returned report says which files have extents that could not be bound to // an archive header. func (i *Importer) verifyYenc(ctx context.Context, b *bundle.Bundle, names []string, stage stageFunc) (rep extentReport, reason Reason, attributable bool, err error) { + rep.unbound = map[int]string{} if i.cfg.Headers == nil { return rep, ReasonProbeInconclusive, false, errors.New("no way to read article headers") } tables := tablesOf(b, names) picks := pickExtents(b) - reserve := extentReserve(b, picks) + partPicks := pickParts(b) + markNeed0(tables, partPicks) + reserve := extentReserve(b, picks) + partReserve(b, partPicks) must, mustBytes := planMust(tables) limit, ceiling := i.cfg.Policy.YencSamples(), i.cfg.Policy.YencBytes() if len(must) > limit || mustBytes+reserve > ceiling { @@ -330,18 +396,25 @@ func (i *Importer) verifyYenc(ctx context.Context, b *bundle.Bundle, names []str errTooManyTables, len(tables), len(must), mustBytes, reserve, limit, ceiling) } bud := newCostBudget(max(yencCap(releaseBytes(tables), ceiling), mustBytes+reserve)) - _ = bud.charge(mustBytes) + _ = bud.chargeMany(mustBytes, len(must)) + defer func() { rep.articles, rep.bytes = bud.articles(), bud.spent() }() ctx, cancel := context.WithTimeout(ctx, i.cfg.Policy.YencTimeoutOrDefault()) defer cancel() - if reason, attributable, err := judge(i.runSamples(ctx, tables, must)); err != nil { + rel := newRelState() + if reason, attributable, err := judge(i.runSamples(ctx, tables, must, rel)); err != nil { return rep, reason, attributable, err } - if len(picks) > 0 { + if len(picks) > 0 || len(partPicks) > 0 { st, r, serr := stage() if serr != nil { return rep, r, false, serr } - if rep, reason, attributable, err = i.verifyExtents(ctx, b, st, picks, bud); err != nil { + if len(picks) > 0 { + if rep, reason, attributable, err = i.verifyExtents(ctx, b, st, picks, bud); err != nil { + return rep, reason, attributable, err + } + } + if reason, attributable, err = i.verifyParts(ctx, b, tables, st, partPicks, rel, bud, &rep); err != nil { return rep, reason, attributable, err } } @@ -349,7 +422,7 @@ func (i *Importer) verifyYenc(ctx context.Context, b *bundle.Bundle, names []str for _, s := range optional { _ = bud.charge(tables[s.table].sizes[s.pos]) } - reason, attributable, err = judge(i.runSamples(ctx, tables, optional)) + reason, attributable, err = judge(i.runSamples(ctx, tables, optional, rel)) return rep, reason, attributable, err } @@ -376,11 +449,57 @@ type sampleVerdict struct { reason Reason err error attributable bool + // hdr is the article's header when it was read. + hdr YencHeader + ok bool +} + +// relState collects, across the runs of one accept, what the sampled articles +// of the parts that are slices of a larger posted file say: where the part's +// first article begins in the posted file (every sample implies it) and the +// posted file's total size. Every sample of one part must agree. +type relState struct { + base map[int]int64 + total map[int]int64 + // first is the 0-based offset of article 0 of a table, when it was sampled. + first map[int]int64 +} + +func newRelState() *relState { + return &relState{base: map[int]int64{}, total: map[int]int64{}, first: map[int]int64{}} +} + +// judgeRelative compares the samples of every relative part with each other. +func (st *relState) judge(tables []sampleTable, plan []sample, res []sampleVerdict) { + for k, s := range plan { + t := &tables[s.table] + if !res[k].ok || res[k].reason != "" { + continue + } + h := res[k].hdr + if s.pos == 0 { + st.first[s.table] = h.PartBegin + } + if !t.rel { + continue + } + base := h.PartBegin - t.prefix[s.pos] + bad := base < 0 || base+t.total > h.FileSize + if b0, ok := st.base[s.table]; ok && (b0 != base || st.total[s.table] != h.FileSize) { + bad = true + } + if bad { + res[k] = sampleVerdict{reason: ReasonYencMismatch, attributable: true, err: fmt.Errorf("%s: article %d begins at %d of %d, which does not fit the other articles of the part (%d bytes in the layout)", + t.name, s.pos, h.PartBegin, h.FileSize, t.total)} + continue + } + st.base[s.table], st.total[s.table] = base, h.FileSize + } } // runSamples reads the planned articles' headers (four at a time) and compares // each with the layout of its table. -func (i *Importer) runSamples(ctx context.Context, tables []sampleTable, plan []sample) []sampleVerdict { +func (i *Importer) runSamples(ctx context.Context, tables []sampleTable, plan []sample, rel *relState) []sampleVerdict { results := make([]sampleVerdict, len(plan)) var wg sync.WaitGroup sem := make(chan struct{}, 4) @@ -399,14 +518,21 @@ func (i *Importer) runSamples(ctx context.Context, tables []sampleTable, plan [] // An optional sample that ran out of time is dropped, not a failure. case err != nil: results[k] = sampleVerdict{reason: ReasonProbeInconclusive, err: fmt.Errorf("%s: %w", t.name, err)} - case h.PartSize != t.sizes[s.pos] || h.PartBegin != t.prefix[s.pos] || h.FileSize != t.total: + case t.rel && h.PartSize != t.sizes[s.pos]: + results[k] = sampleVerdict{reason: ReasonYencMismatch, attributable: true, err: fmt.Errorf("%s: article %d says part %d, layout says %d", t.name, s.pos, h.PartSize, t.sizes[s.pos])} + case !t.rel && (h.PartSize != t.sizes[s.pos] || h.PartBegin != t.prefix[s.pos] || h.FileSize != t.total): results[k] = sampleVerdict{reason: ReasonYencMismatch, attributable: true, err: fmt.Errorf("%s: article %d says part %d at %d of %d, layout says %d at %d of %d", t.name, s.pos, h.PartSize, h.PartBegin, h.FileSize, t.sizes[s.pos], t.prefix[s.pos], t.total)} case s.pos == 0 && t.nzbName != "" && h.Name != "" && !strings.EqualFold(h.Name, t.nzbName): results[k] = sampleVerdict{reason: ReasonYencMismatch, err: fmt.Errorf("%s: first article is named %q, your NZB names this entry %q", t.name, h.Name, t.nzbName)} + default: + results[k] = sampleVerdict{hdr: h, ok: true} } }() } wg.Wait() + if rel != nil { + rel.judge(tables, plan, results) + } return results } diff --git a/internal/peernet/bundleimport/types.go b/internal/peernet/bundleimport/types.go index 7f6ef51ed..fcd18b627 100644 --- a/internal/peernet/bundleimport/types.go +++ b/internal/peernet/bundleimport/types.go @@ -419,6 +419,21 @@ type Result struct { // AlreadyImported is set when every file was already present locally with // the same File ID (nothing was written). AlreadyImported bool + // Cost is what the accept-time article reads cost. + Cost Cost +} + +// Cost reports the accept-time reads of one bundle: the articles and declared +// bytes charged to the shared budget (yEnc header samples, archive header +// reads), and how the archive header checks came out. +type Cost struct { + Articles int + Bytes int64 + // ExtentsBound and ExtentsUnbound count the nested-layout extents that were + // bound to an archive header or could not be; PartsBound and PartsUnbound + // do the same for the slices of multi-part files. + ExtentsBound, ExtentsUnbound int + PartsBound, PartsUnbound int } // Counters is a snapshot of the accept counters. diff --git a/internal/peernet/bundleimport/verify_headers.go b/internal/peernet/bundleimport/verify_headers.go index d21cc6617..dba77f8f6 100644 --- a/internal/peernet/bundleimport/verify_headers.go +++ b/internal/peernet/bundleimport/verify_headers.go @@ -21,31 +21,68 @@ func headerSampleSize(total int) int { } // tablesOfMeta lists the segment tables a stored file reads: its own and each -// shared outer volume. +// shared outer volume. The stored metadata of a file read from several archive +// volumes is one flat segment list that no longer says where the volumes are, so +// the boundaries are recovered from the ranges the way the builder does (a +// segment that starts inside its article, one that ends before its article does, +// one longer than the one before). Each recovered piece is then checked relative +// to where its first sampled article begins in its posted file; a table with no +// boundary is one posted file and is checked as before. func tablesOfMeta(name string, m *metapb.FileMetadata) []sampleTable { var out []sampleTable - add := func(label string, segs []*metapb.SegmentData) { - t := sampleTable{name: label} - for _, s := range segs { - if s == nil || s.Id == "" || holes.IsPlaceholderID(s.Id) { - return // a table with a hole cannot be described by one file - } - t.ids = append(t.ids, s.Id) - t.sizes = append(t.sizes, s.SegmentSize) - t.prefix = append(t.prefix, t.total) - t.total += s.SegmentSize + add := func(label string, segs []*metapb.SegmentData, split bool) { + var cuts []int + if split { + cuts = metaBoundaries(segs) } - if len(t.ids) > 0 { - out = append(out, t) + from := 0 + for k := 0; k <= len(cuts); k++ { + to := len(segs) + if k < len(cuts) { + to = cuts[k] + } + t := sampleTable{name: label, rel: len(cuts) > 0} + if len(cuts) > 0 { + t.name = fmt.Sprintf("%s (part %d of %d)", label, k+1, len(cuts)+1) + } + for _, s := range segs[from:to] { + if s == nil || s.Id == "" || holes.IsPlaceholderID(s.Id) { + return // a table with a hole cannot be described by one file + } + t.ids = append(t.ids, s.Id) + t.sizes = append(t.sizes, s.SegmentSize) + t.prefix = append(t.prefix, t.total) + t.total += s.SegmentSize + } + if len(t.ids) > 0 { + out = append(out, t) + } + from = to } } - add(name, m.SegmentData) + add(name, m.SegmentData, true) for j, o := range m.SharedOuterSources { - add(fmt.Sprintf("%s (volume %d)", name, j+1), o.Segments) + add(fmt.Sprintf("%s (volume %d)", name, j+1), o.Segments, false) } return out } +// metaBoundaries returns the indices at which a flat segment list starts a new +// part. +func metaBoundaries(segs []*metapb.SegmentData) []int { + var cuts []int + for i := 1; i < len(segs); i++ { + a, b := segs[i-1], segs[i] + if a == nil || b == nil { + continue + } + if b.StartOffset > 0 || a.EndOffset < a.SegmentSize-1 || b.SegmentSize > a.SegmentSize { + cuts = append(cuts, i) + } + } + return cuts +} + // VerifyPeerHeaders is the optional extra of the forced full check of a pending // peer import (it implements health.PeerHeaderVerifier). The STAT-only check // cannot see a size or offset lie, so a further random sample of the file's @@ -98,7 +135,7 @@ func (i *Importer) VerifyPeerHeaders(ctx context.Context, filePath string) error } ctx, cancel := context.WithTimeout(ctx, i.cfg.Policy.YencTimeoutOrDefault()) defer cancel() - verdicts := i.runSamples(ctx, tables, plan) + verdicts := i.runSamples(ctx, tables, plan, newRelState()) var inconclusive error for _, v := range verdicts { switch v.reason { diff --git a/internal/peernet/bundleimport/yenc_test.go b/internal/peernet/bundleimport/yenc_test.go index 0e2e16c2b..698528bc4 100644 --- a/internal/peernet/bundleimport/yenc_test.go +++ b/internal/peernet/bundleimport/yenc_test.go @@ -324,3 +324,62 @@ func TestAccept_OptionalSamplesRunningOutOfTimeAreDropped(t *testing.T) { require.Error(t, err) assert.Equal(t, ReasonProbeInconclusive, ReasonOf(err)) } + +// The parts of the main media file are sampled like one table: one interior +// article per 500 segments of the whole file (at least three), spread over the +// parts at random, plus the last article of every part and the first article of +// every part that gets an archive header check. +func TestPlanMust_MainFilePartsAreSampledAsOneTable(t *testing.T) { + parts := func(n, articles int) []sampleTable { + out := make([]sampleTable, n) + for i := range out { + out[i] = bigTable(articles, false) + out[i].part, out[i].mainPart, out[i].index, out[i].need0 = true, true, i, true + } + return out + } + for _, tc := range []struct{ n, articles, interior int }{{50, 140, 14}, {200, 140, 56}, {3, 100, 3}} { + tables := parts(tc.n, tc.articles) + must, _ := planMust(tables) + last, first, interior := 0, 0, 0 + for _, s := range must { + switch s.pos { + case tc.articles - 1: + last++ + case 0: + first++ + default: + interior++ + } + } + assert.Equal(t, tc.n, last, "the last article of every part") + assert.Equal(t, tc.n, first, "the first article of every part that is bound") + assert.Equal(t, tc.interior, interior, "%d parts: one interior article per 500 segments of the file, at least 3", tc.n) + } +} + +// What the multi-part checks cost for stored sets of 100 MB volumes at 768 KB +// articles (140 per volume): the mandatory samples plus one header read per +// checked part. +func TestSamplingCostOfMultiVolumeSets(t *testing.T) { + for _, tc := range []struct { + volumes int + articles int + maxMB float64 + wantArticlesMax int + }{{50, 150, 130, 170}, {200, 150, 300, 390}} { + tables := make([]sampleTable, tc.volumes) + for i := range tables { + tables[i] = bigTable(tc.articles, false) + tables[i].part, tables[i].mainPart, tables[i].index = true, true, i + tables[i].need0 = i < maxPartChecks + } + must, mustBytes := planMust(tables) + checked := min(tc.volumes, maxPartChecks) + articles := len(must) + checked + mb := float64(mustBytes+int64(checked)*768_000) / 1e6 + t.Logf("%d volumes: %d mandatory samples + %d header reads = %d articles, %.0f MB", tc.volumes, len(must), checked, articles, mb) + assert.LessOrEqual(t, mb, tc.maxMB) + assert.LessOrEqual(t, articles, tc.wantArticlesMax) + } +} diff --git a/internal/peernet/bundlex/builder_test.go b/internal/peernet/bundlex/builder_test.go index 4bc571a6d..a881ca89c 100644 --- a/internal/peernet/bundlex/builder_test.go +++ b/internal/peernet/bundlex/builder_test.go @@ -217,3 +217,78 @@ func TestBuildBusyAndTransientErrors(t *testing.T) { t.Fatal("failure not counted") } } + +// storedVolumes is a file stored across archive volumes the way the importer +// writes it: one flat segment list, every volume's slice starting inside its +// first article and ending inside its last. +func storedVolumes(name string, volumes, articles int) (*metapb.FileMetadata, [][]string) { + var lists [][]string + m := &metapb.FileMetadata{Status: metapb.FileStatus_FILE_STATUS_HEALTHY, CreatedAt: 1, ModifiedAt: 2, ReleaseDate: 1700000000, OriginalFilename: name} + for v := 0; v < volumes; v++ { + list := ids(name+"-vol"+string(rune('a'+v)), articles) + lists = append(lists, list) + for i, id := range list { + s := &metapb.SegmentData{Id: id, SegmentSize: segSize, StartOffset: 0, EndOffset: segSize - 1} + if i == 0 { + s.StartOffset = 60 + } + if i == articles-1 { + s.EndOffset = segSize - 21 + } + m.SegmentData = append(m.SegmentData, s) + m.FileSize += s.EndOffset - s.StartOffset + 1 + } + } + return m, lists +} + +func TestBuildMultiVolumeStoredSetBecomesAMultiPartBundle(t *testing.T) { + m, lists := storedVolumes("movie.mkv", 5, 4) + lib := &fakeLib{scheme: database.PeernetSchemeNZB, paths: []string{"movies/R/movie.mkv"}, + files: map[string]*metapb.FileMetadata{"movies/R/movie.mkv": m}, nzb: lists} + rel := releaseHexOfNZB(lib.nzb) + got, err := newTestBuilder(t, lib, NoPatches{}).Build(context.Background(), rel, "Some.Movie.2020.1080p", posted) + if err != nil { + t.Fatalf("a multi-volume stored set must be shareable: %v", err) + } + rid, _ := contentid.ParseID(rel) + bd, err := bundle.Verify(got.Wire, got.ID, rid) + if err != nil { + t.Fatalf("a receiver would reject the bundle: %v", err) + } + if want := []int{4, 4, 4, 4, 4}; len(bd.Files) != 1 || !equalInts(bd.Files[0].Parts, want) { + t.Fatalf("parts = %v, want %v", bd.Files[0].Parts, want) + } + again, err := newTestBuilder(t, lib, NoPatches{}).Build(context.Background(), rel, "Some.Movie.2020.1080p", posted) + if err != nil || !bytes.Equal(again.Wire, got.Wire) { + t.Fatalf("the multi-part build is not deterministic (%v)", err) + } +} + +func equalInts(a, b []int) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func TestBuildUnrecoverablePartLayoutIsCountedNotShareable(t *testing.T) { + m, lists := storedVolumes("movie.mkv", 3, 4) + // a locally patched segment repeats the last article of the set + last := m.SegmentData[len(m.SegmentData)-1] + m.SegmentData = append(m.SegmentData, &metapb.SegmentData{Id: last.Id, SegmentSize: last.SegmentSize, StartOffset: last.StartOffset, EndOffset: last.StartOffset + 9}) + lib := &fakeLib{scheme: database.PeernetSchemeNZB, paths: []string{"p"}, files: map[string]*metapb.FileMetadata{"p": m}, nzb: lists} + b := newTestBuilder(t, lib, NoPatches{}) + _, err := b.Build(context.Background(), releaseHexOfNZB(lists), "T", posted) + if r := notShareableReason(t, err); r != string(bundle.ReasonPartLayout) { + t.Fatalf("reason = %s", r) + } + if b.Status().NotShareable[string(bundle.ReasonPartLayout)] != 1 { + t.Fatalf("not counted: %v", b.Status().NotShareable) + } +} From 38248852ab75c6c077613dd2eb0a1a159d45b9de Mon Sep 17 00:00:00 2001 From: javi11 Date: Sun, 15 Nov 2026 01:38:41 +0100 Subject: [PATCH 5/5] fix(peernet): name bundle files by virtual path and bound part shapes Found by a live two-instance run and a review of the layout changes. - bundle files are named after their virtual path; archive content had no name, so every archive release was announced and then rejected - the publisher runs the receiver's file policy before announcing - real-pipeline test: importer, builder, verify, accept and read-back for plain, generic-container, RAR4/RAR5 single and multi-volume and password-in-NZB releases - parts after the first must start near their article's beginning and parts before the last must end near its end, which rejects files stitched from small windows - the queue row id is resolved after an upsert, fixing history writes for direct peer imports - local-policy rejections refund the per-publisher fetch budget, and the default budget suits browsing - the importer is not installed when a required collaborator is missing - catch-up publishes are counted; status reports header-bound parts - current rules consolidated in the spec --- cmd/altmount/cmd/peernet_bundle_wiring.go | 38 ++- .../cmd/peernet_bundle_wiring_test.go | 18 +- docs/docs/3. Configuration/peer-network.md | 7 +- .../specs/2026-11-12-peer-network-design.md | 45 +++- .../config/peernet/ActivityCard.tsx | 2 + .../components/config/peernet/peernetStats.ts | 21 +- frontend/src/types/peernet.ts | 5 + frontend/tests/peernetStats.test.ts | 25 ++ internal/api/peernet_handlers.go | 9 +- internal/database/queue_addtoqueue_id_test.go | 56 ++++ internal/database/queue_repository.go | 15 +- .../importer/peer_bundle_realpipeline_test.go | 244 ++++++++++++++++++ internal/importer/peer_bundle_service_test.go | 23 ++ .../archivehdr/archivetest/archivetest.go | 26 +- .../archivehdr/archivetest/rar4volume_test.go | 48 ++++ internal/peernet/bridge/bundleimport.go | 9 + internal/peernet/bundle/limits.go | 14 + internal/peernet/bundle/poc_regress_test.go | 169 ++++++++++++ internal/peernet/bundle/validate.go | 35 ++- internal/peernet/bundleimport/accept.go | 18 +- internal/peernet/bundleimport/accept_test.go | 6 +- internal/peernet/bundleimport/filepolicy.go | 8 +- internal/peernet/bundleimport/importer.go | 34 ++- .../peernet/bundleimport/newchecked_test.go | 24 ++ internal/peernet/bundleimport/parts.go | 6 +- internal/peernet/bundleimport/pathobserver.go | 2 +- .../peernet/bundleimport/pathobserver_test.go | 30 +++ internal/peernet/bundleimport/refund_test.go | 43 +++ internal/peernet/bundleimport/types.go | 45 +++- internal/peernet/bundlex/build_bench_test.go | 4 +- internal/peernet/bundlex/builder.go | 10 +- internal/peernet/bundlex/builder_test.go | 34 ++- internal/peernet/bundlex/fetch.go | 8 + internal/peernet/bundlex/fetch_test.go | 28 +- internal/peernet/bundlex/limits.go | 2 +- internal/peernet/service/bundles.go | 24 ++ internal/peernet/service/catchup_test.go | 3 + internal/peernet/service/publishing.go | 2 + internal/testsupport/mediagen/mediagen.go | 56 ++++ 39 files changed, 1132 insertions(+), 64 deletions(-) create mode 100644 internal/database/queue_addtoqueue_id_test.go create mode 100644 internal/importer/peer_bundle_realpipeline_test.go create mode 100644 internal/peernet/archivehdr/archivetest/rar4volume_test.go create mode 100644 internal/peernet/bundle/poc_regress_test.go create mode 100644 internal/peernet/bundleimport/newchecked_test.go create mode 100644 internal/peernet/bundleimport/refund_test.go create mode 100644 internal/testsupport/mediagen/mediagen.go diff --git a/cmd/altmount/cmd/peernet_bundle_wiring.go b/cmd/altmount/cmd/peernet_bundle_wiring.go index 6d1c5be7a..623469074 100644 --- a/cmd/altmount/cmd/peernet_bundle_wiring.go +++ b/cmd/altmount/cmd/peernet_bundle_wiring.go @@ -163,16 +163,24 @@ type peernetBundleHost struct { hasPending func() bool // observe installs the importer's path observer on the metadata service. observe func(metadata.FileObserver) - build func() *bundleimport.Importer + build func() (*bundleimport.Importer, error) getCfg config.ConfigGetter mu sync.Mutex imp *bundleimport.Importer } +// importer returns the bundle importer, building it on first use. It returns +// nil (and logs why) when a required collaborator is missing: such an importer +// is never installed, so callers must treat nil as "feature unavailable". func (h *peernetBundleHost) importer() *bundleimport.Importer { if h.imp == nil { - h.imp = h.build() + imp, err := h.build() + if err != nil { + slog.Error("Peer bundle import is not available", "error", err) + return nil + } + h.imp = imp if h.observe != nil { h.observe(h.imp.PathObserver()) } @@ -205,15 +213,21 @@ func (h *peernetBundleHost) apply(cfg *config.Config) { if !cfg.PeernetEnabled() { h.importerSvc.SetPeerBundleFastPath(nil) if h.hasPending != nil && h.hasPending() { - h.health.SetPeerBundleGate(h.importer()) - h.api.SetPeerBundleLookup(h.lookup) - return + if imp := h.importer(); imp != nil { + h.health.SetPeerBundleGate(imp) + h.api.SetPeerBundleLookup(h.lookup) + return + } } h.health.SetPeerBundleGate(nil) h.api.SetPeerBundleLookup(nil) return } imp := h.importer() + if imp == nil { + h.importerSvc.SetPeerBundleFastPath(nil) + return + } // Pending peer imports must be honoured even before a fetcher exists. h.health.SetPeerBundleGate(imp) h.api.SetPeerBundleLookup(h.lookup) @@ -290,14 +304,8 @@ func newPeernetBundleHost(d bundleWiringDeps, seams *peernetBundleSeams) *peerne n, _, err := state.PendingCounts(context.Background(), time.Time{}) return err != nil || n > 0 }, - build: func() *bundleimport.Importer { - cfg := bundleImporterConfig(d, state, stagingRoot, seams) - if missing := cfg.Missing(); len(missing) > 0 { - // A programming error: the accept pipeline would reject every - // bundle at runtime. Say so loudly instead of failing quietly. - slog.Error("Peer bundle import is wired without required collaborators", "missing", missing) - } - return bundleimport.New(cfg) + build: func() (*bundleimport.Importer, error) { + return bundleimport.NewChecked(bundleImporterConfig(d, state, stagingRoot, seams)) }, } } @@ -381,5 +389,7 @@ func (b *peerOriginBinder) ImportStatus() service.BundleImportStatus { pending, stale := h.Importer().PendingStatus(context.Background()) return service.BundleImportStatus{Pending: int64(pending), Stale: int64(stale), Accepted: int64(c.Accepted), AlreadyImported: int64(c.AlreadyImported), Rejected: rej, FullCheckPassed: int64(c.FullCheckPassed), FullCheckFailed: int64(c.FullCheckFailed), - Announced: int64(c.Announced), BadBundleReports: int64(c.BadBundleReports)} + Announced: int64(c.Announced), BadBundleReports: int64(c.BadBundleReports), + PartsHeaderBound: int64(c.PartsHeaderBound), PartsHeaderUnbound: int64(c.PartsHeaderUnbound), + ExtentsHeaderBound: int64(c.ExtentsHeaderBound), ExtentsHeaderUnbound: int64(c.ExtentsHeaderUnbound)} } diff --git a/cmd/altmount/cmd/peernet_bundle_wiring_test.go b/cmd/altmount/cmd/peernet_bundle_wiring_test.go index 5f26d18b9..ae235b574 100644 --- a/cmd/altmount/cmd/peernet_bundle_wiring_test.go +++ b/cmd/altmount/cmd/peernet_bundle_wiring_test.go @@ -41,7 +41,7 @@ func bundleHostFixture(t *testing.T) (*peernetBundleHost, *recFastPath, *recGate fp, g, l, built := &recFastPath{}, &recGate{}, &recLookup{}, new(int) h := &peernetBundleHost{ importerSvc: fp, health: g, api: l, seams: &peernetBundleSeams{}, receiving: func() bool { return true }, - build: func() *bundleimport.Importer { *built++; return bundleimport.New(bundleimport.Config{}) }, + build: func() (*bundleimport.Importer, error) { *built++; return bundleimport.New(bundleimport.Config{}), nil }, } return h, fp, g, l, built } @@ -136,7 +136,9 @@ func TestPeerOriginBinder_FailsClosedUntilTheImporterExists(t *testing.T) { require.Error(t, err, "unbound") h, _, _, _, _ := bundleHostFixture(t) - h.build = func() *bundleimport.Importer { return bundleimport.New(bundleimport.Config{State: stubBundleState{}}) } + h.build = func() (*bundleimport.Importer, error) { + return bundleimport.New(bundleimport.Config{State: stubBundleState{}}), nil + } b.Bind(h) _, _, err = b.PeerBundleStatus(context.Background(), "peer.mkv") require.Error(t, err, "importer not built yet (feature not applied)") @@ -294,3 +296,15 @@ var ( _ health.PeerBundleGate = (*bundleimport.Importer)(nil) _ health.PeerHeaderVerifier = (*bundleimport.Importer)(nil) ) + +// An importer that cannot be built (a required collaborator is missing) is +// never installed, anywhere. +func TestBundleHost_NeverInstallsAnImporterThatCannotBeBuilt(t *testing.T) { + h, fp, g, l, _ := bundleHostFixture(t) + h.build = func() (*bundleimport.Importer, error) { return bundleimport.NewChecked(bundleimport.Config{}) } + assert.NotPanics(t, func() { h.apply(enabledCfg(true)) }) + assert.Nil(t, fp.cur) + assert.Nil(t, g.cur, "no gate: a nil importer behind the interface would panic at first use") + assert.Nil(t, l.cur) + assert.Nil(t, h.Importer()) +} diff --git a/docs/docs/3. Configuration/peer-network.md b/docs/docs/3. Configuration/peer-network.md index 733ee78c3..fca203568 100644 --- a/docs/docs/3. Configuration/peer-network.md +++ b/docs/docs/3. Configuration/peer-network.md @@ -166,7 +166,7 @@ A **full check** of every segment is then scheduled at high priority. Until it p **What Sonarr and Radarr see while it is pending.** An item they grabbed through the SABnzbd-compatible API stays in the *queue* as work in progress (status `Downloading`, 99 percent) and is not in the *history*, so they never import content that is not verified yet. When the full check passes, the item moves to the history as completed and the ARR imports it as usual. When it fails, or when the 7-day limit below is reached, it appears in the history as **failed** with a short message ("Peer-supplied release failed its full health check and was removed"), so the ARR blocklists the release and searches again. An item you hid from the ARRs yourself (skip ARR notification) stays hidden in every state. The Queue page of AltMount shows the item as completed from the start, because the files are playable at once. -**When a full check fails.** In every case the same single path runs and no ARR repair is ever attempted for the file: the queue item turns into a failed one, and every file of that import is removed (metadata, content index, library copy, health record and peer state), so a corrupt file does not stay in your library and the rest of the release cannot complete the import later. This holds for an item an ARR grabbed, for a release started from Stremio (which no ARR knows) and for an import hidden from the ARRs. Only files of a peer import that is in the pending state are ever removed; nothing else is touched. The publisher is **not trusted for bundle fetches for a while**: 30 minutes, doubling per repeated failure up to 6 hours, forgiven after a day or as soon as a later bundle from the same publisher passes its full check. This never affects record sync. It is mild on purpose: a failure can also mean your own providers lack articles. +**When a full check fails.** A check that finds missing articles is not a verdict yet: like any health check it is retried on the health schedule (every 15 minutes by default, up to the configured maximum number of retries), because articles can still be propagating. Only when the last retry fails is the import failed. A contradiction between the layout and the articles' own headers is different: it fails the import at once. When it fails, in every case the same single path runs and no ARR repair is ever attempted for the file: the queue item turns into a failed one, and every file of that import is removed (metadata, content index, library copy, health record and peer state), so a corrupt file does not stay in your library and the rest of the release cannot complete the import later. This holds for an item an ARR grabbed, for a release started from Stremio (which no ARR knows) and for an import hidden from the ARRs. Only files of a peer import that is in the pending state are ever removed; nothing else is touched. The publisher is **not trusted for bundle fetches for a while**: 30 minutes, doubling per repeated failure up to 6 hours, forgiven after a day or as soon as a later bundle from the same publisher passes its full check. This never affects record sync. It is mild on purpose: a failure can also mean your own providers lack articles. **Stale imports.** A pending import is held for at most 7 days: after that it is failed exactly as above (the publisher is not blamed, since no check ran), and the status page shows how many are pending and how many waited too long. Turning the peer network off does not strand pending imports: the full check keeps running for them until they finish. @@ -189,6 +189,11 @@ Before anything is written the layout is also checked against the articles thems - **Archive extent checks.** For every file read through outer archive volumes, each extent (outer volume, offset, length) is compared with the archive's own file header, read from your providers through the same hidden reader that will serve the file (so an encrypted outer volume is read decrypted with the bundle's key). The header that precedes the extent's data must be a stored (not compressed) file entry, not a service block or a directory and not itself encrypted; its data must start exactly at the extent's offset and be as long as the extent in that volume; the entry's unpacked size must equal the file's size; and its continuation flags must fit the extent's place (first part, middle part, last part). RAR 4, RAR 5 and 7z (plain end header) are understood. When the outer volume is itself a stored file of a wrapping archive and the volume begins in that archive's first article, the wrapping header is checked as well (stored, same size, and for RAR 5 the IV in the header must equal the bundle's). Up to 64 extents are checked (the first and last of the main media file always, the rest chosen at random). A contradiction is attributable to the publisher: the bundle is refused (`extent_mismatch`), the publisher is reported (fetch back-off only) and the layout goes on the negative cache for 7 days. - **Multi-part files (archive volumes).** Each slice of a file stored across volumes is bound the same way: the volume is read from its first article, and a stored, unencrypted (or, for a keyed file, encrypted) file entry's data must start at the byte where the slice starts, be as long as the slice, belong to a file of the declared size, and carry continuation flags that fit the slice's place (first, middle, last). For RAR 5 the volume numbers must also run in the order of the slices, so two middle volumes cannot be swapped. A slice that claims to be a whole posted file of media data must not begin like an archive volume. A second file stored after the first in a volume is bound too when your NZB is at hand (the articles before it are taken to be as long as the first one, which its own offset has to confirm). Up to 64 slices are checked (the first and last of every media file always, the rest chosen at random). A contradiction is attributable: `extent_mismatch`, the publisher is reported, the layout goes on the negative cache. A slice that cannot be judged (a second file in a volume of an old-format import, a split 7-Zip set whose end header is in another volume, encoded 7z headers) is unverifiable and treated like an unverifiable extent below. - **What stays unverifiable.** An extent whose archive cannot be judged is *unverifiable*: an unknown format (a disc image such as a Blu-ray ISO), a 7z end header that is compressed or encrypted, encrypted archive headers, a header walk that exceeds its budget, a damaged header chain, or a missing article. Such a file is accepted only when its media passes the structural check (Matroska, MP4 or transport stream) through the layout; a generic container in that situation is refused (`extent_unverifiable`), because readable windows prove nothing about misplaced bytes. Extents beyond the 64 checked are covered by sampling only. Files stored inline in a single archive volume (no outer volumes) are not extents and are verified by the head and structure checks only. +- **Part shape.** A file stored across archive volumes is described as parts, one per volume. Apart from the first and the last part, a part must start within 64 KiB of its first article and stop within 64 KiB (or one eighth of its articles, for volumes that carry a recovery record) of the end of its last article, so a part can never be a sliver of a large article. A bundle that breaks this is refused whatever the header checks say. +- **File names.** A shared file carries the name it has in your library (the last part of its path). Releases whose files would all be refused by a receiver's file policy (no media file) are not announced; they are counted as not shareable (`no_media_file`). +- **Costs and limits.** Fetching a bundle may be tried 10 times in a burst and 60 times per hour per publisher (and 10 in a burst, 60 per hour for the whole node). A bundle refused by your file policy before any article was read does not use up the publisher's share, and a bundle on the failed-bundle list costs nothing. +- **What is still checked by sampling only.** Order of the volumes is bound only where the format numbers its volumes (RAR 5); for RAR 4 and split 7z sets a swap of two middle volumes is not detected at accept, and at most 64 volumes are header-checked per accept. The forced full check reads further headers. These limits remain until the checks are made exhaustive. +- **Status.** The activity card shows how many parts and nested extents of accepted bundles were tied to their archive header and how many were not. - **Sampling.** The last article of every run of articles (volume or file) is always read: a size lie anywhere in a run shifts every later offset, so the last article's offset and total expose it. The main media file's runs also get their first article and random interior ones (one per 500 articles, at least three). The first and a random interior article of the other runs are read as far as the budget allows, at randomly chosen runs. The budget is one percent of the release size, at least 64 MiB and at most 1 GiB, shared with the archive header reads above and raised if the last articles alone need more; only a release whose last articles exceed 1 GiB (about 1,400 volumes) is refused (`too_many_tables`). Optional samples that run out of the time limit are dropped, mandatory ones are not. A single movie file reads about 5 articles (4 MB), a 50-volume release about 90 (67 MB), a 200-volume release about 280 (215 MB), a 500-volume release about 700 (540 MB), plus a few articles per checked archive volume. A movie stored in a multi-volume archive set reads the last article of every volume, the first article of every checked volume, one article per checked volume for its header and one interior article per 500 articles of the movie: about 165 articles (127 MB) for 50 volumes of 100 MB and about 390 articles (300 MB) for 200 volumes. Two lies that cancel out inside a run are found only when a sampled article lies between them. With your NZB at hand the first article's yEnc file name must also match the NZB entry it belongs to. The forced full check reads a further random sample of headers (1 percent of the articles, 20 to 500) and fails the import on a contradiction. It samples; it does not prove. - **Media structure.** Besides the end of the file, the container is read at random positions inside the file (Matroska: three random 1 MiB windows must contain cluster data; MPEG-TS: eight random packet positions must be in sync; fragmented MP4 must end in a real `moof`+`mdat` pair or `mfra`). What remains unprovable: bytes inside MP4 `mdat`, inside a Matroska frame or between sampled positions. - **Degraded means failed.** A peer release whose full check finds the media playable but with missing segments is failed and removed, never announced. When you supplied the NZB the queue item is not marked failed: it is imported normally from your NZB, once, so you get what a normal import would have produced. A release imported directly (Stremio peer play) is failed. A missing article alone does not count against the publisher (it can be propagation just after posting); only a contradiction between the layout and the articles' own headers does, and that layout is then ignored whoever offers it, for 7 days. diff --git a/docs/superpowers/specs/2026-11-12-peer-network-design.md b/docs/superpowers/specs/2026-11-12-peer-network-design.md index 3aa48ced2..d124b2459 100644 --- a/docs/superpowers/specs/2026-11-12-peer-network-design.md +++ b/docs/superpowers/specs/2026-11-12-peer-network-design.md @@ -462,6 +462,41 @@ above. publisher; the fallback title drops a trailing media extension; the consumers' node-ID waiter makes a final check under its lock. +## Phase 3: current rules (authoritative) + +This section states what the code does today. Where an older section below says +something different, that bullet is marked "(superseded - see Current rules)". + +- **Shareable:** every release the importer produced whose files pass the + receiver's pure file policy (a media file after sanitising, allowed + extensions). The publisher runs that policy at build time and counts a + release that fails it as not shareable (`no_media_file` or the policy's own + reason). A file is named by the basename of its virtual path, never by + `OriginalFilename` (archive-stored content has none). +- **Archive sets:** stored RAR 4, RAR 5 (single and multi-volume) and encrypted + stored RAR 5 with the password in the NZB are shared as parts of one file. + Multi-volume stored sets are no longer "not shareable". +- **Part shape (deterministic, `bundle.Validate`):** in a multi-part file only + the first part may start deep inside its first article and only the last may + end deep inside its last one. Every other part starts within `HeaderSlack` + (64 KiB) of its first article and stops within the larger of `TrailerSlack` + (64 KiB) and one eighth of its articles from the end of its last article. A + part cannot be a sliver of a large article. +- **Open network:** archive layouts are accepted and header-checked like on + private networks; the earlier "plain layouts only" restriction is gone. +- **Fetch budget:** per publisher a burst of 10 and 60 per hour; node-wide + burst 10 and 60 per hour. A bundle the local file policy refused before any + article was read gives its per-publisher attempt back; a failed-bundle cache + hit costs nothing. +- **Status:** the import status reports how many parts and extents of accepted + bundles were bound to an archive header and how many were not. +- **Full check:** missing articles are retried on the health schedule before + the import fails; a header contradiction fails it at once. +- **Still sampling-based (not done in this round):** accept reads at most 64 + header checks, part order is bound only for RAR 5 volume numbers, verification + is charged by declared sizes, the fast-path window is fixed, and the forced + full check re-derives part boundaries from flat metadata. + ## Phase 3: bundle format Implemented by `internal/peernet/bundle`: a pure package (no network, database @@ -985,10 +1020,10 @@ A second review with working PoCs found that the identities bind less than the f **Layout binding.** - A pool entry is referenced at most once within a segment table, and by one table only: two tables may share an entry only when they are the identical table (every reference, size and usable range equal, as for a shared outer volume or a PAR2 file listed by several files). Tables are keyed everywhere (validation, sampling) by `bundle.KeyOf`, a hash of the whole table, never by their first message-id. PoCs P1 (one article referenced 1,999 times as a 1.4 GB file) and P4 (two files sharing a first article, the second never sampled) are regression tests. -- Interior segments of every table are used whole (`Start == 0`, `End == Size-1`); only the first may start inside its article and only the last may end inside it (`ErrPartialRange`). Real archive slicing in this importer is expressed by extents into outer volumes, never by interior partial ranges of an inline table, so no other layout kind was needed. PoC P2 (100 articles of 700 KB each used for 10 bytes) is a regression test. In the files scheme a declared segment size is bounded by 4 MiB (`MaxFilesSchemeSegment`): posters use 0.25 to 1 MB articles (the importer's own estimates assume about 768 KB) and nothing in the importer or the readers expects more; the nzb scheme keeps the 8 MiB wire cap and is additionally bounded by the user's NZB. -- Sampling. Every table gets its first and last article and at least one uniformly random interior article (crypto/rand); tables of media files get one per 500 segments, at least three. The cap is 600 articles and 512 MiB (`MaxYencSamples`, `MaxYencBytes`) within 90 seconds (`YencTimeout`). A layout whose tables do not fit at the minimum rate is rejected (`too_many_tables`), never left unsampled. Measured at 768 KB articles: a single 1.1 GB file reads 5 articles (4 MB); a 50-volume release 150 articles (115 MB); a 200-volume release 600 articles (461 MB); 201 volumes are rejected. When the user's NZB is present the first article's yEnc `name=` must equal the subject file name of the NZB entry the table reads (obfuscated names bind too); a disagreement with the local NZB rejects the bundle but is not reported against the publisher. +- Interior segments of every table are used whole (superseded - see Current rules) (`Start == 0`, `End == Size-1`); only the first may start inside its article and only the last may end inside it (`ErrPartialRange`). Real archive slicing in this importer is expressed by extents into outer volumes, never by interior partial ranges of an inline table, so no other layout kind was needed. PoC P2 (100 articles of 700 KB each used for 10 bytes) is a regression test. In the files scheme a declared segment size is bounded by 4 MiB (`MaxFilesSchemeSegment`): posters use 0.25 to 1 MB articles (the importer's own estimates assume about 768 KB) and nothing in the importer or the readers expects more; the nzb scheme keeps the 8 MiB wire cap and is additionally bounded by the user's NZB. +- Sampling (superseded - see Current rules). Every table gets its first and last article and at least one uniformly random interior article (crypto/rand); tables of media files get one per 500 segments, at least three. The cap is 600 articles and 512 MiB (`MaxYencSamples`, `MaxYencBytes`) within 90 seconds (`YencTimeout`). A layout whose tables do not fit at the minimum rate is rejected (`too_many_tables`), never left unsampled. Measured at 768 KB articles: a single 1.1 GB file reads 5 articles (4 MB); a 50-volume release 150 articles (115 MB); a 200-volume release 600 articles (461 MB); 201 volumes are rejected. When the user's NZB is present the first article's yEnc `name=` must equal the subject file name of the NZB entry the table reads (obfuscated names bind too); a disagreement with the local NZB rejects the bundle but is not reported against the publisher. - Forced full check. A pending peer import additionally gets a header sample proportional to its size (1 percent of its articles, at least 20, at most 500, bounded bytes) through an optional `health.PeerHeaderVerifier` installed together with the peer gate (nil: no change; `internal/health` imports nothing from the peer network). A mismatch fails the import and is the only failure attributable to the publisher. This is sampling, not proof. -- Open network and nested layouts. Archive layouts (outer volumes, extents, AES keys, clips) cannot be bound without reading archive headers. In the open network only plain layouts are accepted (`layout_not_allowed_open`); private networks keep nested layouts, and what a node may serve is unchanged. Open-network bundles for archive releases need archive-header spot checks (planned). +- Open network and nested layouts (superseded - see Current rules). Archive layouts (outer volumes, extents, AES keys, clips) cannot be bound without reading archive headers. In the open network only plain layouts are accepted (`layout_not_allowed_open`); private networks keep nested layouts, and what a node may serve is unchanged. Open-network bundles for archive releases need archive-header spot checks (planned). - Structure check. A fragmented MP4 must end in a `moof`+`mdat` pair whose sizes chain to the end of the file or an `mfra` with a valid `mfro` (optionally followed by `free`/`skip`); a non-fragmented MP4 walks up to 256 top-level box headers instead of a 64-read cap; Matroska reads three random interior positions of 1 MiB (three tries each) and requires two consecutive Cluster-interior elements (SimpleBlock, BlockGroup, Timecode, Cluster) in each; MPEG-TS checks eight random positions on the packet grid. The open network accepts only containers with a real check (Matroska/WebM, MP4/MOV, MPEG-TS/M2TS); other media is refused there (`container_not_verifiable`). Still unprovable: bytes inside `mdat`, inside a Matroska frame or between sampled TS packets, and everything between sampled positions. A forger who puts real structure everywhere the check can look, and garbage elsewhere, is not detected by structure alone. **Trust and fairness.** Corroboration (bundle offers and phase 2 hints, `KnownPrefixes`) keys on the remote IP actually connected to when the peer was last verified (`peernet_peers.connected_ip`, migration 063; /24 for IPv4, /48 for IPv6); a peer with none counts for nothing, so two DNS names for one server are one range. Attesters that rank offers must themselves be known. The fast path hands its requirements (open allowed, corroboration floor) to the fetcher, which filters offers before dialing anyone, so with the switch off no open-network peer is contacted. The serve-side build budget is spent only after the build slot is held and the build starts, and a requester (node ID, public address) can use at most a sixth of it with a burst of one; cache hits and refused requests cost nothing. Only one bundle is verified at a time node-wide (`bundle.VerifyContext`); `MaxDecompressedSize` is 16 MiB (a 50 GB release is about 3.2 MB canonical, so roughly 340,000 segments, about 250 GB at 768 KB per article, still fits comfortably). Decode peak heap fell from 6.5 times the canonical size to about 3.3 times (single exact read buffer, streaming canonical comparison instead of a second full re-encode). @@ -1006,8 +1041,8 @@ Owner decision: every kind of release is accepted on every network. The second r - Wrapper check. When the outer table's first segment starts inside its article (an inner volume sliced out of an outer archive), the table is also read raw from the start of its first article; if that article starts a RAR archive, the entry whose data begins at the table's first byte must be stored, as large as the table, and (RAR 5 file encryption record) carry the bundle's IV. RAR 4 keeps its IV derived from the password and cannot be compared. A wrong key or IV shows as garbage where the inner volume's signature should be. - Bounds. Per scan: 64 blocks, 160 reads, 64 KiB per header, 64 KiB 7z end header; per volume at most 24 distinct articles touched; every touched article is charged (declared size) to the accept-wide budget; at most 64 extents are checked (all of them up to that, else the first and last extent of the main media file plus a crypto/rand choice). No size from the archive is used to allocate; every offset is added with overflow checks; no recursion. Fuzz targets `FuzzVerify` and `FuzzVerifyUnchecked` (checksums off, to reach the parsers below them) run over seed archives built by `archivetest`. - Outcomes. `Bound`; `Unverifiable` (unknown format, encoded 7z header, encrypted archive headers, budget spent, damaged header chain, read error); `Mismatch`. A mismatch is attributable: `extent_mismatch`, fetch-domain report, negative cache. A missing article is a probe error, never a mismatch. -- Accepted without header binding, always only with a structural media check through the layout (a generic container is refused, `extent_unverifiable`): disc-image extents; 7z with an encoded end header; encrypted RAR headers (not shareable on the publisher side anyway); volumes whose walk exceeded the budget; every file whose outer volumes are not a RAR or 7z archive. Extents beyond the 64 checked are covered by sampling only. Inline single-archive files (no extents) are accepted on the head and structure checks, as before. -- Found while studying the producer, not changed: a stored multi-volume RAR or 7z file with no outer encryption is flattened into one inline table with partial ranges at every volume boundary, which `Validate` rejects (`ErrPartialRange`), so such releases are not shareable today; and the yEnc whole-file check (offset zero, table total equals the file total) is only right for tables that cover a posted file from its first to its last article, so a table that is a slice from the middle of a larger posted file is refused by it. +- Accepted without header binding, always only with a structural media check through the layout (a generic container is refused, `extent_unverifiable`): disc-image extents; 7z with an encoded end header; encrypted RAR headers (not shareable on the publisher side anyway); volumes whose walk exceeded the budget; every file whose outer volumes are not a RAR or 7z archive. Extents beyond the 64 checked are covered by sampling only. Inline single-archive files (no extents) are accepted on the head and structure checks, as before (superseded - see Current rules). +- Found while studying the producer (superseded - see Current rules): a stored multi-volume RAR or 7z file with no outer encryption is flattened into one inline table with partial ranges at every volume boundary, which `Validate` rejects (`ErrPartialRange`), so such releases are not shareable today; and the yEnc whole-file check (offset zero, table total equals the file total) is only right for tables that cover a posted file from its first to its last article, so a table that is a slice from the middle of a larger posted file is refused by it. **One budget.** The yEnc header samples and the archive header reads share one budget of declared article bytes (`costBudget`). Order: mandatory samples, archive header checks, then optional samples with what is left. - Mandatory: the last article of every table (a size lie anywhere in a table shifts every later offset, so the last article's offset and the table total expose a single lie with certainty); for the largest media file's tables also the first article and random interior ones (one per 500, at least three). diff --git a/frontend/src/components/config/peernet/ActivityCard.tsx b/frontend/src/components/config/peernet/ActivityCard.tsx index cee4dbb5c..85d45a1aa 100644 --- a/frontend/src/components/config/peernet/ActivityCard.tsx +++ b/frontend/src/components/config/peernet/ActivityCard.tsx @@ -4,6 +4,7 @@ import { backboneHint, describeBreakdown, fetchFailLabel, + headerBindingLine, importRejectLabel, libraryShareLine, notShareableLabel, @@ -123,6 +124,7 @@ function BundlesBlock({ status }: { status: PeernetStatus }) { Awaiting full check: {count(imp?.full_check_passed)} passed,{" "} {count(imp?.full_check_failed)} failed

+ {headerBindingLine(imp) &&

{headerBindingLine(imp)}

} {(imp?.pending ?? 0) > 0 && (

Waiting for a full check: {count(imp?.pending)} diff --git a/frontend/src/components/config/peernet/peernetStats.ts b/frontend/src/components/config/peernet/peernetStats.ts index 5a0934263..3fe4709e1 100644 --- a/frontend/src/components/config/peernet/peernetStats.ts +++ b/frontend/src/components/config/peernet/peernetStats.ts @@ -1,4 +1,4 @@ -import type { PeernetLibraryStatus } from "../../../types/peernet"; +import type { PeernetBundleImportStatus, PeernetLibraryStatus } from "../../../types/peernet"; // Plain-language labels for the machine keys the status endpoint reports. The // lookups use Maps so a key such as "__proto__" can never reach a prototype; @@ -60,6 +60,7 @@ const NOT_SHAREABLE_LABELS: ReadonlyMap = new Map([ ["too_large", "Too large to bundle"], ["no_post_date", "No post date known"], ["no_files", "No files in the release"], + ["no_media_file", "No media file a receiver would accept"], ["self_check_failed", "The built bundle failed its own check"], ["other", "Other reasons"], ]); @@ -341,3 +342,21 @@ export function backboneHint( if (typeof n !== "number" || !Number.isFinite(n) || n <= 0) return null; return "Health records are not shared because no provider backbone is recognised. Set a backbone (storage group) on your providers, or use a provider host that is in the known list."; } + +/** + * One line on how well the files of accepted bundles were tied to their + * archive headers at accept time, or null when nothing was checked yet. "Not + * tied" parts were accepted on structure alone; they are the ones a full check + * matters most for. + */ +export function headerBindingLine(imp: PeernetBundleImportStatus | undefined): string | null { + if (!imp) return null; + const pb = count(imp.parts_header_bound); + const pu = count(imp.parts_header_unbound); + const eb = count(imp.extents_header_bound); + const eu = count(imp.extents_header_unbound); + if (pb + pu + eb + eu === 0) return null; + const parts = `${pb} ${pb === 1 ? "part" : "parts"} tied to their archive header, ${pu} not tied`; + if (eb + eu === 0) return `Archive header checks: ${parts}.`; + return `Archive header checks: ${parts}; ${eb} nested ${eb === 1 ? "extent" : "extents"} tied, ${eu} not tied.`; +} diff --git a/frontend/src/types/peernet.ts b/frontend/src/types/peernet.ts index 43130bb88..5b828903f 100644 --- a/frontend/src/types/peernet.ts +++ b/frontend/src/types/peernet.ts @@ -134,6 +134,11 @@ export interface PeernetBundleImportStatus { pending?: number; /** Pending past the maximum age: failed, never announced. */ stale?: number; + /** Parts (and nested-layout extents) of accepted bundles bound to an archive header, and those that could not be. */ + parts_header_bound?: number; + parts_header_unbound?: number; + extents_header_bound?: number; + extents_header_unbound?: number; } export interface PeernetBundlesStatus { diff --git a/frontend/tests/peernetStats.test.ts b/frontend/tests/peernetStats.test.ts index 6164d2615..c961cc1ab 100644 --- a/frontend/tests/peernetStats.test.ts +++ b/frontend/tests/peernetStats.test.ts @@ -6,6 +6,7 @@ import { fetchFailLabel, formatRelativeTime, hasAnyPublishingLevel, + headerBindingLine, importRejectLabel, levelNeedsConfirmation, libraryShareLine, @@ -85,6 +86,7 @@ const BACKEND_NOT_SHAREABLE_KEYS = [ "too_large", "no_post_date", "no_files", + "no_media_file", "self_check_failed", "other", ]; @@ -434,3 +436,26 @@ describe("backbone hint", () => { expect(backboneHint(undefined)).toBeNull(); }); }); + +describe("headerBindingLine", () => { + test("says nothing before any part or extent was checked", () => { + expect(headerBindingLine(undefined)).toBeNull(); + expect(headerBindingLine({ parts_header_bound: 0, parts_header_unbound: 0 })).toBeNull(); + }); + + test("reports bound and unbound parts and extents", () => { + expect(headerBindingLine({ parts_header_bound: 12, parts_header_unbound: 0 })).toBe( + "Archive header checks: 12 parts tied to their archive header, 0 not tied.", + ); + expect( + headerBindingLine({ + parts_header_bound: 3, + parts_header_unbound: 1, + extents_header_bound: 2, + extents_header_unbound: 0, + }), + ).toBe( + "Archive header checks: 3 parts tied to their archive header, 1 not tied; 2 nested extents tied, 0 not tied.", + ); + }); +}); diff --git a/internal/api/peernet_handlers.go b/internal/api/peernet_handlers.go index 299e2a562..e2a4c6ecc 100644 --- a/internal/api/peernet_handlers.go +++ b/internal/api/peernet_handlers.go @@ -201,6 +201,11 @@ type peernetImportDTO struct { BadBundleReports int64 `json:"bad_bundle_reports"` Pending int64 `json:"pending"` Stale int64 `json:"stale"` + + PartsHeaderBound int64 `json:"parts_header_bound"` + PartsHeaderUnbound int64 `json:"parts_header_unbound"` + ExtentsHeaderBound int64 `json:"extents_header_bound"` + ExtentsHeaderUnbound int64 `json:"extents_header_unbound"` } func copyCounts(m map[string]int64) map[string]int64 { @@ -221,7 +226,9 @@ func toBundlesDTO(b service.BundleStatus) peernetBundlesDTO { Import: peernetImportDTO{Accepted: b.Import.Accepted, AlreadyImported: b.Import.AlreadyImported, Rejected: copyCounts(b.Import.Rejected), FullCheckPassed: b.Import.FullCheckPassed, FullCheckFailed: b.Import.FullCheckFailed, Announced: b.Import.Announced, BadBundleReports: b.Import.BadBundleReports, - Pending: b.Import.Pending, Stale: b.Import.Stale}, + Pending: b.Import.Pending, Stale: b.Import.Stale, + PartsHeaderBound: b.Import.PartsHeaderBound, PartsHeaderUnbound: b.Import.PartsHeaderUnbound, + ExtentsHeaderBound: b.Import.ExtentsHeaderBound, ExtentsHeaderUnbound: b.Import.ExtentsHeaderUnbound}, } } diff --git a/internal/database/queue_addtoqueue_id_test.go b/internal/database/queue_addtoqueue_id_test.go new file mode 100644 index 000000000..91c7b59dd --- /dev/null +++ b/internal/database/queue_addtoqueue_id_test.go @@ -0,0 +1,56 @@ +package database + +import ( + "context" + "database/sql" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// AddToQueue must leave the id of the row with the item's path in item.ID +// whether it inserted, updated or (while the row is pending or processing) +// left the existing row alone; a stale last-insert id or zero would be an +// invalid foreign key for every later write that refers to the item. +func TestAddToQueueResolvesTheExistingRowOnEveryDialect(t *testing.T) { + backends := map[string]func(t *testing.T) *QueueRepository{ + "sqlite": func(t *testing.T) *QueueRepository { + db, err := sql.Open("sqlite3", ":memory:") + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + db.SetMaxOpenConns(1) + setupQueueSchema(t, db) + return NewQueueRepository(db, DialectSQLite) + }, + } + if hasPostgresDSN() { + backends["postgres"] = func(t *testing.T) *QueueRepository { + return NewQueueRepository(openPostgresLatest(t), DialectPostgres) + } + } + for name, mk := range backends { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + r := mk(t) + first := &ImportQueueItem{NzbPath: "peer://a.nzb", Priority: QueuePriorityNormal, Status: QueueStatusCompleted, MaxRetries: 1} + require.NoError(t, r.AddToQueue(ctx, first)) + require.NotZero(t, first.ID) + // Other rows move the last-insert id. + for _, p := range []string{"/x/1.nzb", "/x/2.nzb"} { + require.NoError(t, r.AddToQueue(ctx, &ImportQueueItem{NzbPath: p, Priority: QueuePriorityNormal, Status: QueueStatusPending, MaxRetries: 1})) + } + again := &ImportQueueItem{NzbPath: "peer://a.nzb", Priority: QueuePriorityNormal, Status: QueueStatusCompleted, MaxRetries: 1} + require.NoError(t, r.AddToQueue(ctx, again)) + assert.Equal(t, first.ID, again.ID, "an update resolves to the existing row") + + busy := &ImportQueueItem{NzbPath: "/x/1.nzb", Priority: QueuePriorityNormal, Status: QueueStatusPending, MaxRetries: 1} + require.NoError(t, r.AddToQueue(ctx, busy)) // left alone while pending + require.NoError(t, r.AddToQueue(ctx, &ImportQueueItem{NzbPath: "/x/3.nzb", Priority: QueuePriorityNormal, Status: QueueStatusPending, MaxRetries: 1})) + busy2 := &ImportQueueItem{NzbPath: "/x/1.nzb", Priority: QueuePriorityNormal, Status: QueueStatusPending, MaxRetries: 1} + require.NoError(t, r.AddToQueue(ctx, busy2)) + assert.NotZero(t, busy2.ID) + assert.Equal(t, busy.ID, busy2.ID, "a skipped upsert resolves to the existing row") + }) + } +} diff --git a/internal/database/queue_repository.go b/internal/database/queue_repository.go index 9d7082e9d..9806fc5d5 100644 --- a/internal/database/queue_repository.go +++ b/internal/database/queue_repository.go @@ -232,10 +232,21 @@ func (r *QueueRepository) AddToQueue(ctx context.Context, item *ImportQueueItem) if err != nil { return fmt.Errorf("failed to add queue item: %w", err) } - item.ID, err = result.LastInsertId() - if err != nil { + // LastInsertId is the connection's previous insert when the upsert + // updated an existing row, so it never identifies that row. + if _, err = result.LastInsertId(); err != nil { return fmt.Errorf("failed to get last insert ID: %w", err) } + item.ID = 0 + } + if item.ID == 0 { + // The upsert updated (or, on PostgreSQL, left alone) an existing row: + // the row is the one with this path. + var id int64 + if err := r.db.QueryRowContext(ctx, `SELECT id FROM import_queue WHERE nzb_path = ?`, item.NzbPath).Scan(&id); err != nil { + return fmt.Errorf("failed to resolve queue item %q after upsert: %w", item.NzbPath, err) + } + item.ID = id } item.CreatedAt = time.Now() diff --git a/internal/importer/peer_bundle_realpipeline_test.go b/internal/importer/peer_bundle_realpipeline_test.go new file mode 100644 index 000000000..66b1be259 --- /dev/null +++ b/internal/importer/peer_bundle_realpipeline_test.go @@ -0,0 +1,244 @@ +package importer + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/pbkdf2" + "crypto/sha256" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/javi11/nntppool/v5" + "github.com/kipsilabs/altmount/internal/config" + "github.com/kipsilabs/altmount/internal/database" + "github.com/kipsilabs/altmount/internal/metadata" + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/archivehdr/archivetest" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/bundleimport" + "github.com/kipsilabs/altmount/internal/peernet/bundlex" + "github.com/kipsilabs/altmount/internal/peernet/contentid" + "github.com/kipsilabs/altmount/internal/testsupport/fakepool" + "github.com/kipsilabs/altmount/internal/testsupport/mediagen" + "github.com/kipsilabs/altmount/internal/testsupport/nzbbuild" + "github.com/stretchr/testify/require" +) + +// notShareable lists release types a publisher is allowed to refuse, with the +// reason. A type that is not here must build, verify, be accepted by a +// receiver and read back byte-identically. +var notShareable = map[string]string{} + +type rpProber struct{} + +func (rpProber) ProbeRelease(context.Context, []bundleimport.ProbeFile) error { return nil } + +type rpAnnouncer struct{} + +func (rpAnnouncer) AnnounceVerifiedImport(context.Context, bundleimport.Announcement) error { + return nil +} + +type rpReporter struct{} + +func (rpReporter) ReportBadBundle(context.Context, bundleimport.Origin, string) {} +func (rpReporter) ReportVerifiedBundle(context.Context, bundleimport.Origin, string) {} +func (rpReporter) ReportFailedBundle(context.Context, string, string) {} + +type rpOrigins struct{ fromNZB bool } + +func (o rpOrigins) FromNZB(context.Context, string) (bool, error) { return o.fromNZB, nil } + +type rpLibrary struct{ paths []string } + +func (l rpLibrary) ReleaseFiles(context.Context, string) (int, []string, error) { + return database.PeernetSchemeNZB, l.paths, nil +} + +// rpRelease is one release type of the matrix: the posted NZB files and the +// payload (the media file as the library will show it). +type rpRelease struct { + name string + payload string // library file name of the payload + data []byte + password string + build func(post func(name string, data []byte) nzbbuild.File) []nzbbuild.File +} + +const ( + rpSeg = 40_000 + rpPayload = 500_000 +) + +func rpRar5Volumes(name string, pl []byte, n int) func(func(string, []byte) nzbbuild.File) []nzbbuild.File { + return func(post func(string, []byte) nzbbuild.File) []nzbbuild.File { + vol := (len(pl) + n - 1) / n + var out []nzbbuild.File + for i := 0; i < n; i++ { + lo, hi := i*vol, min(len(pl), (i+1)*vol) + b := archivetest.RAR5Volume(i, archivetest.File{Name: name, Data: pl[lo:hi], SplitBefore: i > 0, SplitAfter: i < n-1, UnpackedSize: int64(len(pl))}) + out = append(out, post(fmt.Sprintf("Release.part%02d.rar", i+1), b.Data)) + } + return out + } +} + +func rpRar4Volumes(name string, pl []byte, n int) func(func(string, []byte) nzbbuild.File) []nzbbuild.File { + return func(post func(string, []byte) nzbbuild.File) []nzbbuild.File { + vol := (len(pl) + n - 1) / n + var out []nzbbuild.File + for i := 0; i < n; i++ { + lo, hi := i*vol, min(len(pl), (i+1)*vol) + b := archivetest.RAR4Volume(i, i == n-1, archivetest.File{Name: name, Data: pl[lo:hi], SplitBefore: i > 0, SplitAfter: i < n-1, UnpackedSize: int64(len(pl))}) + out = append(out, post(fmt.Sprintf("Release.part%02d.rar", i+1), b.Data)) + } + return out + } +} + +func rpEncrypted(name string, pl []byte, pw string) func(func(string, []byte) nzbbuild.File) []nzbbuild.File { + return func(post func(string, []byte) nzbbuild.File) []nzbbuild.File { + salt := make([]byte, 16) + iv := []byte("0123456789abcdef") + key, err := pbkdf2.Key(sha256.New, pw, salt, 1<<15, 32) + if err != nil { + panic(err) + } + blk, _ := aes.NewCipher(key) + padded := append([]byte{}, pl...) + for len(padded)%16 != 0 { + padded = append(padded, 0) + } + cipher.NewCBCEncrypter(blk, iv).CryptBlocks(padded, padded) + b := archivetest.RAR5(archivetest.File{Name: name, Data: padded, IV: iv, UnpackedSize: int64(len(pl))}) + return []nzbbuild.File{post("Release.rar", b.Data)} + } +} + +func rpReleases() []rpRelease { + mkv := mediagen.MKV(rpPayload) + avi := mediagen.AVI(rpPayload) + plain := func(n string, d []byte) func(func(string, []byte) nzbbuild.File) []nzbbuild.File { + return func(post func(string, []byte) nzbbuild.File) []nzbbuild.File { return []nzbbuild.File{post(n, d)} } + } + return []rpRelease{ + {name: "plain mkv", payload: "Movie.2024.mkv", data: mkv, build: plain("Movie.2024.mkv", mkv)}, + {name: "plain avi (generic container)", payload: "Movie.2024.avi", data: avi, build: plain("Movie.2024.avi", avi)}, + {name: "rar5 single volume stored", payload: "Movie.2024.mkv", data: mkv, build: func(post func(string, []byte) nzbbuild.File) []nzbbuild.File { + return []nzbbuild.File{post("Release.rar", archivetest.RAR5(archivetest.File{Name: "Movie.2024.mkv", Data: mkv}).Data)} + }}, + {name: "rar5 multi volume stored (4)", payload: "Movie.2024.mkv", data: mkv, build: rpRar5Volumes("Movie.2024.mkv", mkv, 4)}, + {name: "rar4 single volume stored", payload: "Movie.2024.mkv", data: mkv, build: func(post func(string, []byte) nzbbuild.File) []nzbbuild.File { + return []nzbbuild.File{post("Release.rar", archivetest.RAR4(archivetest.File{Name: "Movie.2024.mkv", Data: mkv}).Data)} + }}, + {name: "rar4 multi volume stored (4)", payload: "Movie.2024.mkv", data: mkv, build: rpRar4Volumes("Movie.2024.mkv", mkv, 4)}, + {name: "rar5 encrypted, password in the NZB", payload: "Movie.2024.mkv", data: mkv, password: "fixture-pw-123", build: rpEncrypted("Movie.2024.mkv", mkv, "fixture-pw-123")}, + } +} + +func rpVirtualPaths(t *testing.T, root string) []string { + t.Helper() + var out []string + require.NoError(t, filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err == nil && !d.IsDir() && strings.HasSuffix(p, ".meta") { + rel, _ := filepath.Rel(root, strings.TrimSuffix(p, ".meta")) + out = append(out, filepath.ToSlash(rel)) + } + return err + })) + return out +} + +func TestRealPipelineEveryReleaseTypeIsShareableAndAccepted(t *testing.T) { + for _, rel := range rpReleases() { + t.Run(rel.name, func(t *testing.T) { + ctx := context.Background() + // Publisher: the real importer over the fake pool. + env := newBatteryEnv(t) + post := func(name string, data []byte) nzbbuild.File { + var segs []nzbbuild.Segment + total := (len(data) + rpSeg - 1) / rpSeg + for i := 0; i < total; i++ { + lo, hi := i*rpSeg, min(len(data), (i+1)*rpSeg) + id := fmt.Sprintf("%s-%s-%03d@rp.test", strings.ReplaceAll(rel.name, " ", ""), name, i+1) + env.client.SetBehavior(id, fakepool.SegmentBehavior{Bytes: data[lo:hi], YEnc: nntppool.YEncMeta{ + FileName: name, FileSize: int64(len(data)), Part: int64(i + 1), Total: int64(total), PartBegin: int64(lo), PartSize: int64(hi - lo)}}) + segs = append(segs, nzbbuild.Segment{ID: id, Bytes: hi - lo}) + } + return nzbbuild.File{Subject: name, Segments: segs} + } + nzb := nzbbuild.Build(rel.build(post)...) + if rel.password != "" { + nzb.Meta = map[string]string{"password": rel.password} + } + _, _, err := env.runImport(nzb, "Release.Name.2024.1080p") + require.NoError(t, err, "the real importer must import the fixture") + paths := rpVirtualPaths(t, env.metaRoot) + require.NotEmpty(t, paths) + + src := bundlex.NewMetadataSource(env.svc, rpOrigins{fromNZB: rel.password != ""}) + lists, err := src.NZBSegmentIDs(paths[0]) + require.NoError(t, err) + store := &metapb.NzbStore{} + for _, l := range lists { + fe := &metapb.NzbFileEntry{} + for _, id := range l { + fe.Segments = append(fe.Segments, &metapb.NzbSeg{Id: id}) + } + store.Files = append(store.Files, fe) + } + rid, err := contentid.ReleaseID(store) + require.NoError(t, err) + + builder := bundlex.NewBuilder(bundlex.BuilderConfig{Source: src, Library: rpLibrary{paths: paths}, Patches: bundlex.NoPatches{}}) + built, err := builder.Build(ctx, rid.String(), "Release Name 2024 1080p", time.Unix(1_700_000_000, 0)) + if reason, ok := notShareable[rel.name]; ok { + require.Error(t, err, "documented not shareable: %s", reason) + return + } + require.NoError(t, err, "a release the importer produced must be shareable") + t.Logf("MATRIX %-40s built=yes scheme=%d wire=%d bytes", rel.name, built.Scheme, len(built.Wire)) + bd, err := bundle.Verify(built.Wire, built.ID, rid) + require.NoError(t, err) + + // Receiver: real accept over the same fake pool, real stager. + lib := metadata.NewMetadataService(t.TempDir()) + db, err := database.NewDB(database.Config{DatabasePath: filepath.Join(t.TempDir(), "db")}) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + cfg := config.DefaultConfig() + pm := processorTestPoolManager{client: env.client} + stager := bundleimport.NewFSStager(bundleimport.FSStagerDeps{Pool: pm, Config: func() *config.Config { return cfg }, Root: t.TempDir()}) + imp := bundleimport.New(bundleimport.Config{ + Library: lib, Stager: stager, Prober: rpProber{}, Headers: bundleimport.NewPoolHeaderReader(pm), + State: database.NewPeernetBundleImportRepository(db.Connection(), database.DialectSQLite), Announcer: rpAnnouncer{}, Reporter: rpReporter{}, + HealthEnabled: func() bool { return true }, + }) + res, err := imp.Accept(ctx, bundleimport.Request{Bundle: bd, Target: bundleimport.Target{Dir: "movies/Release.Name.2024.1080p", Category: "movies"}}) + require.NoError(t, err, "a receiver must accept what the publisher announced") + require.Len(t, res.Paths, 1) + require.True(t, strings.HasSuffix(res.Paths[0], "/"+rel.payload), "accepted file %q, want name %q", res.Paths[0], rel.payload) + + stage, err := stager.NewStaging() + require.NoError(t, err) + defer stage.Close() + m, err := lib.ReadFileMetadata(res.Paths[0]) + require.NoError(t, err) + require.NoError(t, stage.WriteFileMetadata(res.Paths[0], m)) + f, err := stage.Opener().OpenFile(ctx, res.Paths[0], os.O_RDONLY, 0) + require.NoError(t, err) + defer f.Close() + got, err := io.ReadAll(f) + require.NoError(t, err) + require.Equal(t, sha256.Sum256(rel.data), sha256.Sum256(got), "accepted file bytes differ from the payload") + t.Logf("MATRIX %-40s accepted=yes bytes-identical=yes size=%d", rel.name, len(got)) + }) + } +} diff --git a/internal/importer/peer_bundle_service_test.go b/internal/importer/peer_bundle_service_test.go index 4a8627335..628d93b0e 100644 --- a/internal/importer/peer_bundle_service_test.go +++ b/internal/importer/peer_bundle_service_test.go @@ -238,3 +238,26 @@ func TestPeerImportDirRefusesTitlesThatEscapeTheCategoryFolder(t *testing.T) { } assert.Equal(t, catDir+"/Some Movie", s.PeerImportDir("Movies", "Some Movie.mkv")) } + +// A second direct import of the same release finds its queue row completed, +// so the upsert changes nothing; the import must still be tied to that row +// (not to a stale or zero id) and its history entry must be written. +func TestRecordPeerImportTwiceKeepsTheQueueReferenceValid(t *testing.T) { + ctx := context.Background() + s := newPeerService(t) + rec := PeerImportRecord{Title: "My Release", Category: "movies", ResultingPath: "movies/My Release", + WrittenPaths: []string{"movies/My Release/a.mkv"}, TotalSize: 99, ReleaseID: "abcdef0123456789abcdef"} + first, err := s.RecordPeerImport(ctx, rec) + require.NoError(t, err) + // Unrelated inserts on the connection make a stale last-insert id differ. + for i := 0; i < 3; i++ { + addItem(t, s, false) + } + second, err := s.RecordPeerImport(ctx, rec) + require.NoError(t, err) + assert.Equal(t, first, second, "the same release resolves to the same queue row") + var n int + require.NoError(t, s.database.Connection().QueryRowContext(ctx, + `SELECT COUNT(*) FROM import_history WHERE nzb_id = ?`, first).Scan(&n)) + assert.GreaterOrEqual(t, n, 1, "history written against the queue row") +} diff --git a/internal/peernet/archivehdr/archivetest/archivetest.go b/internal/peernet/archivehdr/archivetest/archivetest.go index 68c43ffda..76a13a053 100644 --- a/internal/peernet/archivehdr/archivetest/archivetest.go +++ b/internal/peernet/archivehdr/archivetest/archivetest.go @@ -129,7 +129,16 @@ func rar5(volume int, files []File) Built { } // RAR4 builds a RAR 4 archive. -func RAR4(files ...File) Built { +func RAR4(files ...File) Built { return rar4(-1, false, files) } + +// RAR4Volume builds volume (0-based) of a multi-volume RAR 4 set, one that a +// real RAR reader follows: the main header carries the volume and +// new-numbering flags (and first-volume on volume 0), and the end-of-archive +// block says whether more volumes follow. A file continued into the next +// volume must set SplitAfter, one continued from the previous SplitBefore. +func RAR4Volume(volume int, last bool, files ...File) Built { return rar4(volume, last, files) } + +func rar4(volume int, last bool, files []File) Built { var buf bytes.Buffer buf.Write([]byte{'R', 'a', 'r', '!', 0x1A, 0x07, 0x00}) header := func(kind byte, flags uint16, body []byte) { @@ -141,7 +150,14 @@ func RAR4(files ...File) Built { binary.LittleEndian.PutUint16(data[:2], uint16(crc32.ChecksumIEEE(data[2:]))) buf.Write(data) } - header(0x73, 0, make([]byte, 6)) + mainFlags := uint16(0) + if volume >= 0 { + mainFlags = 0x0001 | 0x0010 + if volume == 0 { + mainFlags |= 0x0100 + } + } + header(0x73, mainFlags, make([]byte, 6)) var offsets []int64 for _, f := range files { body := make([]byte, 25) @@ -173,7 +189,11 @@ func RAR4(files ...File) Built { offsets = append(offsets, int64(buf.Len())) buf.Write(f.Data) } - header(0x7B, 0, nil) + endFlags := uint16(0) + if volume >= 0 && !last { + endFlags = 0x0001 + } + header(0x7B, endFlags, nil) return Built{Data: buf.Bytes(), DataOffsets: offsets} } diff --git a/internal/peernet/archivehdr/archivetest/rar4volume_test.go b/internal/peernet/archivehdr/archivetest/rar4volume_test.go new file mode 100644 index 000000000..443209b0e --- /dev/null +++ b/internal/peernet/archivehdr/archivetest/rar4volume_test.go @@ -0,0 +1,48 @@ +package archivetest_test + +import ( + "encoding/binary" + "testing" + + "github.com/kipsilabs/altmount/internal/peernet/archivehdr/archivetest" +) + +// flagsOf returns the flags of the block of the given type in a RAR 4 volume +// (headers only; the test volumes carry little data). +func flagsOf(t *testing.T, data []byte, kind byte) uint16 { + t.Helper() + for pos := 7; pos+7 <= len(data); { + size := int(binary.LittleEndian.Uint16(data[pos+5 : pos+7])) + flags := binary.LittleEndian.Uint16(data[pos+3 : pos+5]) + if data[pos+2] == kind { + return flags + } + if flags&0x8000 != 0 && kind != data[pos+2] { + size += int(binary.LittleEndian.Uint32(data[pos+7 : pos+11])) + } + pos += size + } + t.Fatalf("no block of type %#x", kind) + return 0 +} + +// A multi-volume RAR 4 set has the volume and new-numbering flags on the +// main header (first-volume on volume 0) and "more volumes follow" on every +// end block but the last; the real RAR reader needs all of them. +func TestRAR4VolumeFlags(t *testing.T) { + f := archivetest.File{Name: "a.mkv", Data: []byte("x"), SplitAfter: true, UnpackedSize: 2} + first := archivetest.RAR4Volume(0, false, f) + if got := flagsOf(t, first.Data, 0x73); got != 0x0111 { + t.Fatalf("first volume main flags %#x", got) + } + if got := flagsOf(t, first.Data, 0x7B); got != 0x0001 { + t.Fatalf("first volume end flags %#x", got) + } + last := archivetest.RAR4Volume(1, true, archivetest.File{Name: "a.mkv", Data: []byte("y"), SplitBefore: true, UnpackedSize: 2}) + if got := flagsOf(t, last.Data, 0x73); got != 0x0011 { + t.Fatalf("last volume main flags %#x", got) + } + if got := flagsOf(t, last.Data, 0x7B); got != 0 { + t.Fatalf("last volume end flags %#x", got) + } +} diff --git a/internal/peernet/bridge/bundleimport.go b/internal/peernet/bridge/bundleimport.go index 695fe59ce..35aa2b517 100644 --- a/internal/peernet/bridge/bundleimport.go +++ b/internal/peernet/bridge/bundleimport.go @@ -313,6 +313,15 @@ func (r bundleReporter) ReportBadBundle(_ context.Context, o bundleimport.Origin r.s.NoteBundleFullCheck(o.NetworkID, o.PublisherID, false) } +// RefundFetch gives the publisher's fetch budget back when the service can. +func (r bundleReporter) RefundFetch(_ context.Context, o bundleimport.Origin) { + if rf, ok := r.s.(interface { + RefundBundleFetch(networkID, publisherID string) + }); ok { + rf.RefundBundleFetch(o.NetworkID, o.PublisherID) + } +} + // ReportFailedBundle puts a layout that failed local verification on the // negative cache when the service keeps one. func (r bundleReporter) ReportFailedBundle(_ context.Context, bundleID, releaseID string) { diff --git a/internal/peernet/bundle/limits.go b/internal/peernet/bundle/limits.go index b321a3c26..7feebc134 100644 --- a/internal/peernet/bundle/limits.go +++ b/internal/peernet/bundle/limits.go @@ -22,6 +22,20 @@ const ( // segment table (main files, shared outer sources and PAR2 files), and // separately the message-ID pool. MaxSegments = 500_000 + // HeaderSlack is the most a part after the first may start into its first + // article. What precedes a continuation volume's data is the archive + // signature, the main header and one file header: a few hundred bytes for + // RAR 4, RAR 5 and 7z, a few KiB with a 2 KiB name and the extra records + // (times, hash, encryption). 64 KiB leaves a wide margin and still makes a + // 700 KB article give up at most about 9 percent to headers. + HeaderSlack = 64 << 10 + // TrailerSlack is the most a part before the last may stop short of the end + // of its last article. A volume ends with an end-of-archive block (tens of + // bytes); a RAR recovery record (-rr) stored in the volume can be larger, + // up to about ten percent of it, so the bound is the larger of TrailerSlack + // and 1/TrailerShareDivisor of the part's articles (12.5 percent). + TrailerSlack = 64 << 10 + TrailerShareDivisor = 8 // MaxFiles caps the files of one bundle. MaxFiles = 10_000 // MaxReleaseFiles caps the NZB file entries of the "nzb" scheme release diff --git a/internal/peernet/bundle/poc_regress_test.go b/internal/peernet/bundle/poc_regress_test.go new file mode 100644 index 000000000..5356d8fce --- /dev/null +++ b/internal/peernet/bundle/poc_regress_test.go @@ -0,0 +1,169 @@ +package bundle_test + +import ( + "errors" + "fmt" + "testing" + "time" + + metapb "github.com/kipsilabs/altmount/internal/metadata/proto" + "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/contentid" +) + +func mid(i int) string { return fmt.Sprintf("part%d@poc.example", i) } + +// seal fills the identities and returns the Verify result. +func seal(t *testing.T, b *bundle.Bundle) error { + b.Title, b.PostedUnix = "Poc.Release", 1700000000 + var sum int64 + for i := range b.Files { + b.Files[i].FileID = contentid.ID{byte(i + 1)} + sum += b.Files[i].Size + } + b.TotalSize = sum + // provisional ReleaseID so ToLocal's Validate passes + b.ReleaseID = contentid.ID{9} + loc, err := bundle.ToLocal(b, time.Now()) + if err != nil { + return fmt.Errorf("validate: %w", err) + } + ids := []contentid.ID{} + for i, lf := range loc.Files { + id, err := contentid.FileID(lf.Meta) + if err != nil { + return err + } + b.Files[i].FileID = id + ids = append(ids, id) + } + sortFiles(b) + ids = ids[:0] + for i := range b.Files { + ids = append(ids, b.Files[i].FileID) + } + if b.Scheme == bundle.SchemeFiles { + b.ReleaseID = bundle.FilesReleaseID(ids) + } else { + store := &metapb.NzbStore{} + for _, f := range loc.NZBFiles { + fe := &metapb.NzbFileEntry{} + for _, id := range f { + fe.Segments = append(fe.Segments, &metapb.NzbSeg{Id: id}) + } + store.Files = append(store.Files, fe) + } + b.ReleaseID, err = contentid.ReleaseID(store) + if err != nil { + return err + } + } + wire, id, err := bundle.Pack(b) + if err != nil { + return fmt.Errorf("pack: %w", err) + } + _, err = bundle.Verify(wire, id, b.ReleaseID) + return err +} + +func sortFiles(b *bundle.Bundle) { + for i := 0; i < len(b.Files); i++ { + for j := i + 1; j < len(b.Files); j++ { + if string(b.Files[j].FileID[:]) < string(b.Files[i].FileID[:]) { + b.Files[i], b.Files[j] = b.Files[j], b.Files[i] + } + } + } +} + +// Q1: files scheme, a file made of 100 one-segment parts that each take 10 +// bytes from the middle of a 700 KB article. Header binding at accept would +// be optional for the attacker, so the format itself must refuse it: a part +// that is neither the first nor the last must read nearly all of its articles. +func TestQ1FilesSchemeSingleSegmentPartsAreRefused(t *testing.T) { + n := 100 + pool := []string{} + segs := []bundle.Segment{} + parts := []int{} + for i := 0; i < n; i++ { + pool = append(pool, mid(i)) + segs = append(segs, bundle.Segment{Ref: int64(i), Size: 700000, Start: 1000, End: 1009}) + parts = append(parts, 1) + } + b := &bundle.Bundle{Scheme: bundle.SchemeFiles, Pool: pool, + Files: []bundle.File{{Name: "Movie.mkv", Size: int64(n) * 10, Segments: segs, Parts: parts}}} + if err := seal(t, b); !errors.Is(err, bundle.ErrPartialRange) { + t.Fatalf("want ErrPartialRange, got %v", err) + } +} + +// Q2: the same 100 windows as ONE part must stay refused. +func TestQ2OnePartOfPartialSegmentsIsRefused(t *testing.T) { + n := 100 + pool := []string{} + segs := []bundle.Segment{} + for i := 0; i < n; i++ { + pool = append(pool, mid(i)) + segs = append(segs, bundle.Segment{Ref: int64(i), Size: 700000, Start: 1000, End: 1009}) + } + b := &bundle.Bundle{Scheme: bundle.SchemeFiles, Pool: pool, + Files: []bundle.File{{Name: "Movie.mkv", Size: int64(n) * 10, Segments: segs, Parts: []int{n}}}} + if err := seal(t, b); err == nil { + t.Fatal("a single part of 100 partial segments must be refused") + } +} + +// Q3: nzb scheme, 100 entries; the file takes one arbitrary 10-byte window of +// one article of each, in reverse NZB order. +func TestQ3NZBReorderedWindowsAreRefused(t *testing.T) { + entries := 100 + pool := []string{} + rel := []int{} + for e := 0; e < entries; e++ { + rel = append(rel, 3) + for k := 0; k < 3; k++ { + pool = append(pool, mid(e*3+k)) + } + } + segs := []bundle.Segment{} + parts := []int{} + for e := entries - 1; e >= 0; e-- { + segs = append(segs, bundle.Segment{Ref: int64(e*3 + 1), Size: 700000, Start: 5, End: 14}) + parts = append(parts, 1) + } + b := &bundle.Bundle{Scheme: bundle.SchemeNZB, Pool: pool, ReleaseFiles: rel, + Files: []bundle.File{{Name: "Movie.mkv", Size: int64(entries) * 10, Segments: segs, Parts: parts}}} + if err := seal(t, b); !errors.Is(err, bundle.ErrPartialRange) { + t.Fatalf("want ErrPartialRange, got %v", err) + } +} + +// An honest stored multi-volume layout still validates: each volume is read +// from just after its archive headers to just before its end block. +func TestHonestVolumePartsStillValidate(t *testing.T) { + const art = 700000 + pool := []string{} + segs := []bundle.Segment{} + parts := []int{} + var size int64 + for v := 0; v < 4; v++ { + for k := 0; k < 3; k++ { + ref := int64(len(pool)) + pool = append(pool, mid(int(ref))) + s := bundle.Segment{Ref: ref, Size: art, Start: 0, End: art - 1} + if k == 0 { + s.Start = 300 // headers of the volume + } + if k == 2 { + s.End = art - 1 - 24 // end-of-archive block + } + segs = append(segs, s) + size += s.End - s.Start + 1 + } + parts = append(parts, 3) + } + b := &bundle.Bundle{Scheme: bundle.SchemeFiles, Pool: pool, Files: []bundle.File{{Name: "Movie.mkv", Size: size, Segments: segs, Parts: parts}}} + if err := seal(t, b); err != nil { + t.Fatalf("an honest layout must validate: %v", err) + } +} diff --git a/internal/peernet/bundle/validate.go b/internal/peernet/bundle/validate.go index e4fa65cd8..6e45735ef 100644 --- a/internal/peernet/bundle/validate.go +++ b/internal/peernet/bundle/validate.go @@ -488,7 +488,10 @@ func (v *validator) checkSlicedTable(f *File) (int64, error) { seen := make(map[int64]struct{}, len(segs)) var covered int64 at := 0 - for _, n := range f.Parts { + for pi, n := range f.Parts { + if err := checkPartSlack(segs[at:at+n], pi, len(f.Parts)); err != nil { + return 0, err + } for k := 0; k < n; k++ { i, s := at+k, segs[at+k] first, last := k == 0, k == n-1 @@ -509,6 +512,36 @@ func (v *validator) checkSlicedTable(f *File) (int64, error) { return covered, nil } +// checkPartSlack applies the deterministic part rules: a part of an archive +// volume set is read from just after the volume's headers to just before its +// end block, so, apart from the two ends of the file, no part may begin deep +// inside its first article or stop long before the end of its last one. A +// part that did could be a sliver of a large article, which no header check +// can be required to notice. The first part may begin deep (a file stored +// after another one in the same volume) and the last may end deep (another +// file follows it); every other boundary is bounded by HeaderSlack and +// TrailerSlack (see their documentation). +func checkPartSlack(part []Segment, pi, nParts int) error { + if nParts < 2 { + return nil + } + first, last := part[0], part[len(part)-1] + if pi > 0 && first.Start > HeaderSlack { + return fmt.Errorf("%w: part %d starts %d bytes into its first article", ErrPartialRange, pi, first.Start) + } + if pi < nParts-1 { + var articles int64 + for _, s := range part { + articles += s.Size + } + tail := last.Size - 1 - last.End + if tail > max(TrailerSlack, articles/TrailerShareDivisor) { + return fmt.Errorf("%w: part %d stops %d bytes before the end of its last article", ErrPartialRange, pi, tail) + } + } + return nil +} + // shareArticle records the use of a pool entry by a segment of a part and // refuses every use the boundary exception does not cover. func (v *validator) shareArticle(i int, s Segment, boundary bool) error { diff --git a/internal/peernet/bundleimport/accept.go b/internal/peernet/bundleimport/accept.go index be9099700..b69d34997 100644 --- a/internal/peernet/bundleimport/accept.go +++ b/internal/peernet/bundleimport/accept.go @@ -59,6 +59,16 @@ func (i *Importer) rejected(r Reason, err error) (*Result, error) { // media probe through the layout in a hidden staging area, and only then the // local write in pending state. Any rejection leaves nothing behind. func (i *Importer) Accept(ctx context.Context, req Request) (*Result, error) { + res, err := i.accept(ctx, req) + if err != nil && localPolicyReason(ReasonOf(err)) && req.Origin.PublisherID != "" { + if rf, ok := i.cfg.Reporter.(FetchRefunder); ok { + rf.RefundFetch(ctx, req.Origin) + } + } + return res, err +} + +func (i *Importer) accept(ctx context.Context, req Request) (*Result, error) { ctx, cancel := context.WithTimeout(ctx, i.cfg.Policy.AcceptTimeout) defer cancel() b := req.Bundle @@ -175,7 +185,13 @@ func (i *Importer) Accept(ctx context.Context, req Request) (*Result, error) { if err != nil { return i.rejected(ReasonWriteFailed, err) } - i.cnt.add(func(c *counters) { c.accepted++ }) + i.cnt.add(func(c *counters) { + c.accepted++ + c.partsBound += uint64(res.Cost.PartsBound) + c.partsUnbound += uint64(res.Cost.PartsUnbound) + c.extBound += uint64(res.Cost.ExtentsBound) + c.extUnbound += uint64(res.Cost.ExtentsUnbound) + }) return res, nil } diff --git a/internal/peernet/bundleimport/accept_test.go b/internal/peernet/bundleimport/accept_test.go index 25a757d91..98d312fc2 100644 --- a/internal/peernet/bundleimport/accept_test.go +++ b/internal/peernet/bundleimport/accept_test.go @@ -49,7 +49,11 @@ func (a *fakeAnnouncer) AnnounceVerifiedImport(_ context.Context, an Announcemen return nil } -type fakeReporter struct{ got, verified, failedBundles []string } +type fakeReporter struct{ got, verified, failedBundles, refunds []string } + +func (r *fakeReporter) RefundFetch(_ context.Context, o Origin) { + r.refunds = append(r.refunds, o.PublisherID) +} func (r *fakeReporter) ReportFailedBundle(_ context.Context, bundleID, release string) { r.failedBundles = append(r.failedBundles, bundleID+"/"+release) diff --git a/internal/peernet/bundleimport/filepolicy.go b/internal/peernet/bundleimport/filepolicy.go index 45171a7c1..790dee5b7 100644 --- a/internal/peernet/bundleimport/filepolicy.go +++ b/internal/peernet/bundleimport/filepolicy.go @@ -55,7 +55,13 @@ func (i *Importer) selectFiles(b *bundle.Bundle) ([]int, Reason, error) { if i.cfg.AllowedExtensions != nil { allowed = i.cfg.AllowedExtensions() } - pol := i.cfg.Policy + return SelectFiles(b, allowed, i.cfg.Policy) +} + +// SelectFiles is the receiver's pure file policy: it needs no network or +// state, so a publisher can run it before announcing a bundle that every +// receiver would certainly refuse. +func SelectFiles(b *bundle.Bundle, allowed []string, pol Policy) ([]int, Reason, error) { var keep []int media, sidecars := 0, 0 for n := range b.Files { diff --git a/internal/peernet/bundleimport/importer.go b/internal/peernet/bundleimport/importer.go index 670f10853..d785920e0 100644 --- a/internal/peernet/bundleimport/importer.go +++ b/internal/peernet/bundleimport/importer.go @@ -2,14 +2,18 @@ package bundleimport import ( "errors" + "fmt" "golang.org/x/sync/singleflight" + "strings" "sync" "time" ) -// Config holds the collaborators of an Importer. Fetcher, Recorder, Announcer -// and Reporter may be nil: the matching step is then skipped (no fetch, no -// direct import, nothing announced, nothing reported). +// Config holds the collaborators of an Importer. Library, Stager, Prober, +// State, Headers and HealthEnabled are required (see Missing; NewChecked +// refuses a Config without them). Index, Fetcher, Recorder, Announcer and +// Reporter may be nil: the matching step is then skipped (nothing indexed, no +// fetch, no direct import, nothing announced, nothing reported). type Config struct { Library Library Stager Stager @@ -35,11 +39,10 @@ type Config struct { Now func() time.Time } -// Missing lists the collaborators the accept pipeline cannot work without and -// that are nil. A production wiring must call it and refuse to run with any -// missing (a nil Headers, for example, rejects every bundle at the first -// sample): optional pieces are only Config.AllowedExtensions and -// Config.WillBeChecked (nil means "no restriction" and "every path"). +// Missing lists the required collaborators that are nil. Without one the +// accept pipeline cannot work (a nil State or Library panics at first use, a +// nil Headers rejects every bundle at its first sample). The optional ones are +// listed on Config. func (c Config) Missing() []string { var out []string add := func(nilDep bool, name string) { @@ -51,12 +54,7 @@ func (c Config) Missing() []string { add(c.Stager == nil, "Stager") add(c.Prober == nil, "Prober") add(c.State == nil, "State") - add(c.Index == nil, "Index") - add(c.Fetcher == nil, "Fetcher") add(c.Headers == nil, "Headers") - add(c.Recorder == nil, "Recorder") - add(c.Announcer == nil, "Announcer") - add(c.Reporter == nil, "Reporter") add(c.HealthEnabled == nil, "HealthEnabled") return out } @@ -100,6 +98,16 @@ type targetLock struct { refs int } +// NewChecked is New that refuses a Config with a required collaborator +// missing: an importer built without one would panic or reject everything at +// first use. +func NewChecked(cfg Config) (*Importer, error) { + if missing := cfg.Missing(); len(missing) > 0 { + return nil, fmt.Errorf("bundleimport: required collaborators missing: %s", strings.Join(missing, ", ")) + } + return New(cfg), nil +} + // New builds an Importer. A zero Policy means DefaultPolicy. func New(cfg Config) *Importer { if cfg.Policy == (Policy{}) { diff --git a/internal/peernet/bundleimport/newchecked_test.go b/internal/peernet/bundleimport/newchecked_test.go new file mode 100644 index 000000000..45514c779 --- /dev/null +++ b/internal/peernet/bundleimport/newchecked_test.go @@ -0,0 +1,24 @@ +package bundleimport + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// An importer is never built without a collaborator it cannot work without: +// a nil State or Library would panic at first use. +func TestNewCheckedRefusesAMissingRequiredCollaborator(t *testing.T) { + imp, err := NewChecked(Config{}) + require.Error(t, err) + assert.Nil(t, imp) + for _, name := range []string{"Library", "Stager", "Prober", "State", "Headers", "HealthEnabled"} { + assert.Contains(t, err.Error(), name) + } + e := newEnv(t) + cfg := e.imp.cfg + cfg.Prober = e.prober + _, err = NewChecked(cfg) + require.NoError(t, err, "optional collaborators (Index, Fetcher, Recorder) may be nil") +} diff --git a/internal/peernet/bundleimport/parts.go b/internal/peernet/bundleimport/parts.go index 1402b5218..f394b614d 100644 --- a/internal/peernet/bundleimport/parts.go +++ b/internal/peernet/bundleimport/parts.go @@ -215,7 +215,11 @@ func (i *Importer) verifyParts(ctx context.Context, b *bundle.Bundle, tables []s for n, r := range picks { f := &b.Files[r.file] ps := partsOfSliced(f)[r.part] - w := &partWork{ref: r, segs: ps, raw: rawView(ps), name: fmt.Sprintf("part-%d.bin", n), upTo: ps[0].Start, offset: ps[0].Start, table: byTable[[2]int{r.file, r.part}]} + tbl, ok := byTable[[2]int{r.file, r.part}] + if !ok { + return ReasonInvalid, false, fmt.Errorf("part %d of file %d has no sample table", r.part, r.file) + } + w := &partWork{ref: r, segs: ps, raw: rawView(ps), name: fmt.Sprintf("part-%d.bin", n), upTo: ps[0].Start, offset: ps[0].Start, table: tbl} if r.kind == partRaw { w.upTo = 0 } else { diff --git a/internal/peernet/bundleimport/pathobserver.go b/internal/peernet/bundleimport/pathobserver.go index 39b448723..072127d28 100644 --- a/internal/peernet/bundleimport/pathobserver.go +++ b/internal/peernet/bundleimport/pathobserver.go @@ -11,7 +11,7 @@ import ( // renameGrace is how long a deleted file's row waits for the matching write // (the metadata service reports a rename as a delete followed by a write). -const renameGrace = 3 * time.Second +var renameGrace = 3 * time.Second // PathObserver keeps the peer-bundle state consistent with the library when // the user deletes, moves or renames a file or folder: a deleted file loses its diff --git a/internal/peernet/bundleimport/pathobserver_test.go b/internal/peernet/bundleimport/pathobserver_test.go index 93b7e9621..9603fa834 100644 --- a/internal/peernet/bundleimport/pathobserver_test.go +++ b/internal/peernet/bundleimport/pathobserver_test.go @@ -3,6 +3,7 @@ package bundleimport import ( "context" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -67,3 +68,32 @@ func TestPathObserver_RenamedFolderMovesRows(t *testing.T) { obs.DirectoryDeleted(ctx, "movies/Other") assert.Zero(t, e.stateRows()) } + +// The library reports deletes under whatever spelling the caller used (the +// HTTP API passes a leading slash); the row is keyed by the clean path. +func TestPathObserver_DeleteWithLeadingSlashRemovesTheRow(t *testing.T) { + e, res := newObserved(t) + ctx := context.Background() + obs := e.imp.PathObserver() + require.NoError(t, e.lib.DeleteFileMetadata(ctx, res.Paths[0])) + obs.FileDeleted(ctx, "/"+res.Paths[0]) + obs.Flush() + _, ok, _ := e.repo.Get(ctx, res.Paths[0]) + assert.False(t, ok, "the deleted file's row is gone") +} + +// Through the real metadata service: a user deleting the file ends its state +// row once the rename grace has passed, with no further attempt needed. +func TestPathObserver_UserDeleteThroughTheLibraryEndsTheStateRow(t *testing.T) { + old := renameGrace + renameGrace = 50 * time.Millisecond + t.Cleanup(func() { renameGrace = old }) + e, res := newObserved(t) + ctx := context.Background() + e.lib.SetSecondaryFileObserver(e.imp.PathObserver()) + require.NoError(t, e.lib.DeleteFileMetadata(ctx, res.Paths[0])) + require.Eventually(t, func() bool { + _, ok, _ := e.repo.Get(ctx, res.Paths[0]) + return !ok + }, 2*time.Second, 20*time.Millisecond, "the row of a deleted file must not outlive it") +} diff --git a/internal/peernet/bundleimport/refund_test.go b/internal/peernet/bundleimport/refund_test.go new file mode 100644 index 000000000..d28c50698 --- /dev/null +++ b/internal/peernet/bundleimport/refund_test.go @@ -0,0 +1,43 @@ +package bundleimport + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A bundle the local file policy refuses before any article is read gives the +// publisher's fetch budget back; one that failed on article evidence does not. +func TestAcceptRefundsTheFetchBudgetOnlyForLocalPolicyRejections(t *testing.T) { + e := newEnv(t) + b := e.mkBundle("refund", fileSpec{name: "data.bin", plain: junkPlain(512), segSize: 128}) + _, err := e.imp.Accept(context.Background(), e.req(b)) + require.Error(t, err) + assert.Equal(t, ReasonNoMediaFile, ReasonOf(err)) + assert.Len(t, e.rep.refunds, 1, "no article was read: the attempt is refunded") + + e2 := newEnv(t) + b2 := e2.mkBundle("fail", fileSpec{name: "movie.mkv", plain: junkPlain(512), segSize: 128}) + _, err = e2.imp.Accept(context.Background(), e2.req(b2)) + require.Error(t, err) + assert.NotEqual(t, ReasonNoMediaFile, ReasonOf(err)) + assert.Empty(t, e2.rep.refunds, "an attempt that read articles keeps its charge") +} + +// The import counters say how many parts and extents of accepted bundles were +// bound to an archive header and how many could not be. +func TestCountersExposeHeaderBinding(t *testing.T) { + e := newEnv(t) + payload := mkvPlain(1500) + s := e.mkStoredSet(setSpec{tag: "hb", kind: "rar5", payload: payload, cuts: evenCuts(len(payload), 3), scheme: 1, extraVol: -1}) + res, err := e.imp.Accept(context.Background(), e.req(s.b)) + require.NoError(t, err) + require.Equal(t, 3, res.Cost.PartsBound) + c := e.imp.Counters() + assert.Equal(t, uint64(3), c.PartsHeaderBound) + assert.Equal(t, uint64(res.Cost.PartsUnbound), c.PartsHeaderUnbound) + assert.Equal(t, uint64(res.Cost.ExtentsBound), c.ExtentsHeaderBound) + assert.Equal(t, uint64(res.Cost.ExtentsUnbound), c.ExtentsHeaderUnbound) +} diff --git a/internal/peernet/bundleimport/types.go b/internal/peernet/bundleimport/types.go index fcd18b627..4d655b066 100644 --- a/internal/peernet/bundleimport/types.go +++ b/internal/peernet/bundleimport/types.go @@ -445,17 +445,24 @@ type Counters struct { FullCheckFailed uint64 Announced uint64 BadBundleReports uint64 + // PartsHeaderBound and PartsHeaderUnbound (ExtentsHeaderBound and + // ExtentsHeaderUnbound for nested-layout extents) add up, over the + // accepted bundles, the parts bound to an archive header and those that + // could not be. + PartsHeaderBound, PartsHeaderUnbound uint64 + ExtentsHeaderBound, ExtentsHeaderUnbound uint64 } type counters struct { - mu sync.Mutex - accepted uint64 - already uint64 - rejected map[Reason]uint64 - passed uint64 - failed uint64 - announce uint64 - reports uint64 + partsBound, partsUnbound, extBound, extUnbound uint64 + mu sync.Mutex + accepted uint64 + already uint64 + rejected map[Reason]uint64 + passed uint64 + failed uint64 + announce uint64 + reports uint64 } func (c *counters) add(f func(*counters)) { @@ -472,7 +479,8 @@ func (c *counters) snapshot() Counters { rej[k] = v } return Counters{Accepted: c.accepted, AlreadyImported: c.already, Rejected: rej, FullCheckPassed: c.passed, - FullCheckFailed: c.failed, Announced: c.announce, BadBundleReports: c.reports} + FullCheckFailed: c.failed, Announced: c.announce, BadBundleReports: c.reports, + PartsHeaderBound: c.partsBound, PartsHeaderUnbound: c.partsUnbound, ExtentsHeaderBound: c.extBound, ExtentsHeaderUnbound: c.extUnbound} } // YencSamples returns the sampling bound with its default. @@ -499,3 +507,22 @@ func (p Policy) YencTimeoutOrDefault() time.Duration { } return p.YencTimeout } + +// FetchRefunder is an optional capability of the Reporter: it gives the +// per-publisher fetch budget back for a bundle the local policy refused +// before any article was read (see localPolicyReason). +type FetchRefunder interface { + RefundFetch(ctx context.Context, o Origin) +} + +// localPolicyReason says a rejection came from the local rules alone: it cost +// the network nothing beyond the bundle itself and says nothing about the +// publisher, so it must not use up the publisher's fetch budget. +func localPolicyReason(r Reason) bool { + switch r { + case ReasonTooLarge, ReasonTooManyFiles, ReasonForbiddenFile, ReasonSidecarTooLarge, ReasonNoMediaFile, + ReasonHealthDisabled, ReasonBadTarget, ReasonCollision, ReasonNZBMismatch: + return true + } + return false +} diff --git a/internal/peernet/bundlex/build_bench_test.go b/internal/peernet/bundlex/build_bench_test.go index 825dd6d72..c57bf5f95 100644 --- a/internal/peernet/bundlex/build_bench_test.go +++ b/internal/peernet/bundlex/build_bench_test.go @@ -17,8 +17,8 @@ func BenchmarkBuild75kSegments(b *testing.B) { for i := range video { video[i] = fmt.Sprintf("part%d-%05dof75000.%x@news.example.net", i, i, uint64(i)*2654435761) } - lib := &fakeLib{scheme: database.PeernetSchemeNZB, paths: []string{"p"}, - files: map[string]*metapb.FileMetadata{"p": plainFile("big.mkv", video)}, nzb: [][]string{video}} + lib := &fakeLib{scheme: database.PeernetSchemeNZB, paths: []string{"p/big.mkv"}, + files: map[string]*metapb.FileMetadata{"p/big.mkv": plainFile("big.mkv", video)}, nzb: [][]string{video}} rel := releaseHexOfNZB(lib.nzb) bl := NewBuilder(BuilderConfig{Source: lib, Library: lib, Patches: NoPatches{}}) b.ResetTimer() diff --git a/internal/peernet/bundlex/builder.go b/internal/peernet/bundlex/builder.go index 839e0dab8..79f2a5745 100644 --- a/internal/peernet/bundlex/builder.go +++ b/internal/peernet/bundlex/builder.go @@ -5,6 +5,8 @@ import ( "encoding/hex" "errors" "fmt" + "path" + "strings" "sync" "time" @@ -13,6 +15,7 @@ import ( metapb "github.com/kipsilabs/altmount/internal/metadata/proto" "github.com/kipsilabs/altmount/internal/nzbname" "github.com/kipsilabs/altmount/internal/peernet/bundle" + "github.com/kipsilabs/altmount/internal/peernet/bundleimport" "github.com/kipsilabs/altmount/internal/peernet/contentid" "github.com/kipsilabs/altmount/internal/peernet/records" ) @@ -30,6 +33,7 @@ const ( ReasonNoPostDate = "no_post_date" ReasonNoFiles = "no_files" ReasonSelfCheckFailed = "self_check_failed" + ReasonNoMediaFile = "no_media_file" maxBuiltCacheBytes = 32 << 20 builtTTL = 10 * time.Minute defaultBuildWait = 30 * time.Second @@ -415,7 +419,7 @@ func (b *Builder) build(ctx context.Context, releaseHex, title string, posted ti if len(m.KnownHoles) > 0 { return nil, notShareable(ReasonKnownHoles) } - fi := bundle.FileInput{Meta: m, PatchedLocally: b.anyPatched(m)} + fi := bundle.FileInput{Meta: m, Name: path.Base(strings.ReplaceAll(p, "\\", "/")), PatchedLocally: b.anyPatched(m)} pw, fromNZB := b.cfg.Source.NZBPassword(p, m) fi.PasswordFromNZB = fromNZB if fromNZB && in.NZBPassword == "" { @@ -447,6 +451,10 @@ func (b *Builder) build(ctx context.Context, releaseHex, title string, posted ti if bd.ReleaseID != rid { return nil, notShareable(ReasonReleaseMismatch) } + if _, reason, perr := bundleimport.SelectFiles(bd, nil, bundleimport.Policy{}); perr != nil { + // A receiver applies this same policy before reading any article. + return nil, notShareable(string(reason)) + } wire, id, err := bundle.Pack(bd) if err != nil { if errors.Is(err, bundle.ErrTooLarge) { diff --git a/internal/peernet/bundlex/builder_test.go b/internal/peernet/bundlex/builder_test.go index a881ca89c..3cab6aca1 100644 --- a/internal/peernet/bundlex/builder_test.go +++ b/internal/peernet/bundlex/builder_test.go @@ -67,7 +67,7 @@ func TestBuildFilesSchemeNeverFallsBackFromNZB(t *testing.T) { f := plainFile("a.mkv", ids("a", 5)) fid, _ := contentid.FileID(f) frid, _ := contentid.ReleaseIDFromFiles([]contentid.ID{fid}) - lib2 := &fakeLib{scheme: database.PeernetSchemeFiles, paths: []string{"p"}, files: map[string]*metapb.FileMetadata{"p": f}} + lib2 := &fakeLib{scheme: database.PeernetSchemeFiles, paths: []string{"p/a.mkv"}, files: map[string]*metapb.FileMetadata{"p/a.mkv": f}} got, err := newTestBuilder(t, lib2, NoPatches{}).Build(context.Background(), frid.String(), "T", posted) if err != nil || got.Scheme != bundle.SchemeFiles { t.Fatalf("files scheme: %v %v", got, err) @@ -292,3 +292,35 @@ func TestBuildUnrecoverablePartLayoutIsCountedNotShareable(t *testing.T) { t.Fatalf("not counted: %v", b.Status().NotShareable) } } + +// An archive-stored file has no OriginalFilename in its metadata; the bundle +// must carry the file's virtual file name instead. +func TestBuildNamesFilesByVirtualPath(t *testing.T) { + vid := ids("vid", 20) + lib := &fakeLib{scheme: database.PeernetSchemeNZB, paths: []string{"movies/R/Movie.2020.mkv"}, + files: map[string]*metapb.FileMetadata{"movies/R/Movie.2020.mkv": plainFile("", vid)}, nzb: [][]string{vid}} + rel := releaseHexOfNZB(lib.nzb) + got, err := newTestBuilder(t, lib, NoPatches{}).Build(context.Background(), rel, "T", posted) + if err != nil { + t.Fatal(err) + } + rid, _ := contentid.ParseID(rel) + bd, err := bundle.Verify(got.Wire, got.ID, rid) + if err != nil { + t.Fatal(err) + } + if bd.Files[0].Name != "Movie.2020.mkv" { + t.Fatalf("file name = %q, want the virtual file name", bd.Files[0].Name) + } +} + +// A release the receiver's file policy would certainly refuse is not announced. +func TestBuildRefusesReleaseWithoutMediaFile(t *testing.T) { + d := ids("d", 20) + lib := &fakeLib{scheme: database.PeernetSchemeNZB, paths: []string{"movies/R/payload.bin"}, + files: map[string]*metapb.FileMetadata{"movies/R/payload.bin": plainFile("", d)}, nzb: [][]string{d}} + _, err := newTestBuilder(t, lib, NoPatches{}).Build(context.Background(), releaseHexOfNZB(lib.nzb), "T", posted) + if r := notShareableReason(t, err); r != ReasonNoMediaFile { + t.Fatalf("reason = %s", r) + } +} diff --git a/internal/peernet/bundlex/fetch.go b/internal/peernet/bundlex/fetch.go index 782991415..a565bb92f 100644 --- a/internal/peernet/bundlex/fetch.go +++ b/internal/peernet/bundlex/fetch.go @@ -690,3 +690,11 @@ func (f *Fetcher) ReportVerifiedBundle(nw netid.NetworkID, p identity.NodeID) { defer f.mu.Unlock() delete(f.fcStrikes, strikeKey(nw, p)) } + +// RefundFetch gives one attempt back to the per-publisher budget of p. The +// caller uses it when the bundle was refused by the local file policy alone, +// before any article was read: such an attempt must not eat into the budget of +// later, legitimate plays. The node-wide budget is not refunded. +func (f *Fetcher) RefundFetch(nw netid.NetworkID, p identity.NodeID) { + f.pub.give(strikeKey(nw, p), 1) +} diff --git a/internal/peernet/bundlex/fetch_test.go b/internal/peernet/bundlex/fetch_test.go index 649dd1ad2..a86dff92c 100644 --- a/internal/peernet/bundlex/fetch_test.go +++ b/internal/peernet/bundlex/fetch_test.go @@ -133,8 +133,8 @@ func TestFetchVerifiesAndReportsSource(t *testing.T) { func TestFetchRejectsSubstitutedGarbageOversizedAndStrikes(t *testing.T) { // A valid bundle of another release, served under the announced IDs. - lib2 := &fakeLib{scheme: database.PeernetSchemeNZB, paths: []string{"x"}, nzb: [][]string{ids("z", 4)}} - lib2.files = map[string]*metapb.FileMetadata{"x": plainFile("z.mkv", ids("z", 4))} + lib2 := &fakeLib{scheme: database.PeernetSchemeNZB, paths: []string{"x/z.mkv"}, nzb: [][]string{ids("z", 4)}} + lib2.files = map[string]*metapb.FileMetadata{"x/z.mkv": plainFile("z.mkv", ids("z", 4))} other, err := newTestBuilder(t, lib2, NoPatches{}).Build(context.Background(), releaseHexOfNZB(lib2.nzb), "T", posted) if err != nil { t.Fatal(err) @@ -602,3 +602,27 @@ func TestFetchReportsPublisherLimitAndBackOffAsTheirOwnErrors(t *testing.T) { t.Fatalf("a backed-off publisher is ErrBackedOff (still an ErrNoOffers): %v", err) } } + +// A bundle the local policy refused before any article was read does not use +// up the publisher's budget. +func TestRefundFetchGivesThePublisherBudgetBack(t *testing.T) { + e := newFetchEnv(t, FetchLimits{PerPublisherPerHour: 1, PerPublisherBurst: 1}) + addr, id := honest(t, e) + e.publisher(addr, id, time.Hour) + if _, _, err := e.f.Fetch(context.Background(), e.rel); err != nil { + t.Fatal(err) + } + e.f.RefundFetch(nwA, id) + if _, _, err := e.f.Fetch(context.Background(), e.rel); err != nil { + t.Fatalf("a refunded attempt must leave the budget intact: %v", err) + } +} + +// The defaults must let a user browse and play: a burst of ten and a minute +// per attempt sustained, per publisher. +func TestDefaultFetchLimitsSuitBrowsing(t *testing.T) { + d := DefaultFetchLimits() + if d.PerPublisherBurst < 10 || d.PerPublisherPerHour < 60 { + t.Fatalf("per publisher: burst %d, %d/hour", d.PerPublisherBurst, d.PerPublisherPerHour) + } +} diff --git a/internal/peernet/bundlex/limits.go b/internal/peernet/bundlex/limits.go index e126e4130..75d99c6f7 100644 --- a/internal/peernet/bundlex/limits.go +++ b/internal/peernet/bundlex/limits.go @@ -101,7 +101,7 @@ type FetchLimits struct { // DefaultFetchLimits returns the defaults. func DefaultFetchLimits() FetchLimits { return FetchLimits{ - MaxAttempts: 3, GlobalPerHour: 60, GlobalBurst: 10, PerPublisherPerHour: 12, PerPublisherBurst: 3, + MaxAttempts: 3, GlobalPerHour: 60, GlobalBurst: 10, PerPublisherPerHour: 60, PerPublisherBurst: 10, MaxConcurrent: 2, Timeout: 60 * time.Second, BackoffBase: 10 * time.Minute, BackoffMax: 2 * time.Hour, SoftBackoff: 5 * time.Minute, FullCheckBackoffBase: 30 * time.Minute, FullCheckBackoffMax: 6 * time.Hour, FullCheckDecay: 24 * time.Hour, diff --git a/internal/peernet/service/bundles.go b/internal/peernet/service/bundles.go index eeae1f659..78e38d6e7 100644 --- a/internal/peernet/service/bundles.go +++ b/internal/peernet/service/bundles.go @@ -82,6 +82,11 @@ type BundleImportStatus struct { Rejected map[string]int64 FullCheckPassed, FullCheckFailed int64 Announced, BadBundleReports int64 + // PartsHeaderBound and PartsHeaderUnbound (and the extent pair) say how + // many parts and nested-layout extents of accepted bundles were bound to + // an archive header at accept time and how many could not be. + PartsHeaderBound, PartsHeaderUnbound int64 + ExtentsHeaderBound, ExtentsHeaderUnbound int64 // Pending files still owe their full check; Stale are pending past the // maximum age (they are failed, never announced). Pending, Stale int64 @@ -314,6 +319,25 @@ func (s *Service) FetchBundleWith(ctx context.Context, releaseID string, opts bu return n.fetcher.FetchWith(ctx, releaseID, opts) } +// RefundBundleFetch gives one attempt back to the per-publisher fetch budget +// (a bundle refused by the local file policy before any article was read). +// Unparseable identifiers are ignored. +func (s *Service) RefundBundleFetch(networkID, publisherID string) { + n := s.currentNode() + if n == nil || n.fetcher == nil { + return + } + nw, err := netid.ParseNetworkID(networkID) + if err != nil { + return + } + pub, err := identity.ParseNodeID(publisherID) + if err != nil { + return + } + n.fetcher.RefundFetch(nw, pub) +} + // NoteBundleFullCheck tells the fetcher how the full local check of a release // imported from publisherID's bundle in networkID ended. A failure backs that // publisher off for bundle fetches only (never record sync, never a purge); a diff --git a/internal/peernet/service/catchup_test.go b/internal/peernet/service/catchup_test.go index a1958fb6d..6427f8e39 100644 --- a/internal/peernet/service/catchup_test.go +++ b/internal/peernet/service/catchup_test.go @@ -29,6 +29,9 @@ func TestCatchUpPublishesHealthyIndexedReleasesNeverPublished(t *testing.T) { if _, ok, _ := f.state.Get(ctx, nw, database.PeernetRecordKindRelease, pubRelease, ""); !ok { t.Fatal("the healthy indexed release must be published by the catch-up pass") } + if f.status(t).Publish.Published["release"] == 0 { + t.Fatalf("a catch-up publish must be counted as published: %+v", f.status(t).Publish.Published) + } if _, ok, _ := f.state.Get(ctx, nw, database.PeernetRecordKindAttestation, pubRelease, pubFile); ok { t.Fatal("an attestation is never published without a new check") } diff --git a/internal/peernet/service/publishing.go b/internal/peernet/service/publishing.go index 3225390ad..b7f355d66 100644 --- a/internal/peernet/service/publishing.go +++ b/internal/peernet/service/publishing.go @@ -567,6 +567,8 @@ func (p *pipeline) reannounceOne(ctx context.Context, t publish.Tracked, relHex p.stats.mu.Lock() p.stats.reannounced++ p.stats.mu.Unlock() + // A re-announce and a catch-up publish are records sent: count them as such. + p.stats.add(p.stats.published, kindName(records.KindRelease)) return true } diff --git a/internal/testsupport/mediagen/mediagen.go b/internal/testsupport/mediagen/mediagen.go new file mode 100644 index 000000000..a612cb33e --- /dev/null +++ b/internal/testsupport/mediagen/mediagen.go @@ -0,0 +1,56 @@ +// Package mediagen builds small, structurally valid media files for tests. +package mediagen + +// MKV returns a structurally valid Matroska file of exactly size bytes +// (size >= 200): EBML header, a Segment ending at the file end, a SeekHead +// pointing at Info and at Cues (the last element), and a Cluster of +// SimpleBlocks between them. +func MKV(size int) []byte { + u32 := func(v int) []byte { return []byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)} } + seek := func(id []byte, pos int) []byte { + body := append(append([]byte{0x53, 0xAB, 0x84}, id...), append([]byte{0x53, 0xAC, 0x84}, u32(pos)...)...) + return append([]byte{0x4D, 0xBB, byte(0x80 | len(body))}, body...) + } + info := []byte{0x15, 0x49, 0xA9, 0x66} + cues := []byte{0x1C, 0x53, 0xBB, 0x6B} + ebml := append([]byte{0x1A, 0x45, 0xDF, 0xA3, 0x8B, 0x42, 0x82, 0x88}, "matroska"...) + segStart := len(ebml) + 12 + seekHead := append([]byte{0x11, 0x4D, 0x9B, 0x74, byte(0x80 | 34)}, append(seek(info, 39), seek(cues, size-segStart-9)...)...) + out := append([]byte{}, ebml...) + segSize := size - segStart + out = append(out, 0x18, 0x53, 0x80, 0x67, 0x01, 0, byte(segSize>>40), byte(segSize>>32), byte(segSize>>24), byte(segSize>>16), byte(segSize>>8), byte(segSize)) + out = append(out, seekHead...) + out = append(out, append(append([]byte{}, info...), 0x84, 0, 0, 0, 0)...) + fill := size - len(out) - 12 - 9 + out = append(out, 0x1F, 0x43, 0xB6, 0x75, 0x01, 0, byte(fill>>40), byte(fill>>32), byte(fill>>24), byte(fill>>16), byte(fill>>8), byte(fill)) + out = append(out, SimpleBlocks(fill)...) + return append(out, append(append([]byte{}, cues...), 0x84, 0, 0, 0, 0)...) +} + +// AVI returns a RIFF/AVI container of size bytes: the container signature and +// header list followed by filler. +func AVI(size int) []byte { + out := append([]byte("RIFF\x00\x00\x00\x00AVI LIST\x00\x00\x00\x00hdrlavih"), SimpleBlocks(size-28)...) + n := uint32(len(out) - 8) + out[4], out[5], out[6], out[7] = byte(n), byte(n>>8), byte(n>>16), byte(n>>24) + return out +} + +// SimpleBlocks returns exactly n bytes (n >= 7) of consecutive Matroska +// SimpleBlock elements, what the body of a real Cluster looks like. +func SimpleBlocks(n int) []byte { + out := make([]byte, 0, n) + for k := 0; len(out) < n; k++ { + remaining := n - len(out) + want := min(remaining, 90+(k*37)%300) + if remaining-want < 7 { + want = remaining + } + body := want - 3 + out = append(out, 0xA3, 0x40|byte(body>>8), byte(body), 0x81, byte(k>>8), byte(k), 0x80) + for j := 0; j < body-4; j++ { + out = append(out, byte(1+(len(out)*7+k)%250)) + } + } + return out +}