From df4635ccb99ec7f741613fbff6ebcdc558ac33d3 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Wed, 4 Nov 2026 11:10:25 -0600 Subject: [PATCH 1/4] Honor disabled BigQuery integration tests --- kumo/testing/bigquery.py | 2 + test/testing/test_bigquery_credentials.py | 87 +++++++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/kumo/testing/bigquery.py b/kumo/testing/bigquery.py index db7ad10be..9e5674335 100644 --- a/kumo/testing/bigquery.py +++ b/kumo/testing/bigquery.py @@ -114,6 +114,8 @@ def withBigQueryCredentials(func: Optional[Callable] = None, def decorator(inner_func: Callable): def _setup(): + if os.getenv('KUMO_RUN_GCP_TESTS') == '0': + pytest.skip('BigQuery tests disabled by KUMO_RUN_GCP_TESTS=0') setup_bigquery_credentials(service_account_key) if (not os.getenv(ENV_GOOGLE_CREDENTIALS_JSON) and not os.path.exists(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH)): diff --git a/test/testing/test_bigquery_credentials.py b/test/testing/test_bigquery_credentials.py index a87107373..57ac35c2d 100644 --- a/test/testing/test_bigquery_credentials.py +++ b/test/testing/test_bigquery_credentials.py @@ -141,3 +141,90 @@ def test_connector_loads_external_account_and_impersonates( 'provider': 'bigquery', 'credential': target_credential, } + + +@pytest.mark.parametrize('asynchronous', [False, True]) +@pytest.mark.parametrize('service_account_key', [ + None, + testing_bigquery.FAIL_TABLE_PERMISSION_BIGQUERY_KEY, + testing_bigquery.FAIL_PROJECT_PERMISSION_BIGQUERY_KEY, +]) +def test_disabled_bigquery_skips( + monkeypatch, tmp_path, asynchronous, service_account_key, +): + from io_utils import fsutil + + credentials_path = str(tmp_path / 'adc.json') + fsutil.write_text(credentials_path, '{"type": "external_account"}') + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', credentials_path) + monkeypatch.setenv('KUMO_RUN_GCP_TESTS', '0') + monkeypatch.delenv( + testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT, raising=False) + monkeypatch.delenv( + testing_bigquery.ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT, raising=False) + setup = mock.Mock(wraps=testing_bigquery.setup_bigquery_credentials) + monkeypatch.setattr(testing_bigquery, 'setup_bigquery_credentials', setup) + body = mock.Mock() + + async def async_body(): + body() + + decorated = testing_bigquery.withBigQueryCredentials( + async_body if asynchronous else body, + service_account_key=service_account_key, + ) + with pytest.raises(pytest.skip.Exception, match='KUMO_RUN_GCP_TESTS=0'): + if asynchronous: + import asyncio + + asyncio.run(decorated()) + else: + decorated() + setup.assert_not_called() + body.assert_not_called() + + +@pytest.mark.parametrize('run_gcp', [None, '1']) +@pytest.mark.parametrize('service_account_key, target_env', [ + (testing_bigquery.FAIL_TABLE_PERMISSION_BIGQUERY_KEY, + testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT), + (testing_bigquery.FAIL_PROJECT_PERMISSION_BIGQUERY_KEY, + testing_bigquery.ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT), +]) +def test_enabled_bigquery_requires_target( + monkeypatch, tmp_path, run_gcp, service_account_key, target_env, +): + from io_utils import fsutil + + credentials_path = str(tmp_path / 'adc.json') + fsutil.write_text(credentials_path, '{"type": "external_account"}') + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', credentials_path) + monkeypatch.delenv(target_env, raising=False) + if run_gcp is None: + monkeypatch.delenv('KUMO_RUN_GCP_TESTS', raising=False) + else: + monkeypatch.setenv('KUMO_RUN_GCP_TESTS', run_gcp) + body = mock.Mock() + decorated = testing_bigquery.withBigQueryCredentials( + body, service_account_key=service_account_key) + + with pytest.raises(RuntimeError, match=target_env): + decorated() + body.assert_not_called() + + +@pytest.mark.parametrize('run_gcp', [None, '1']) +def test_enabled_bigquery_runs(monkeypatch, run_gcp): + if run_gcp is None: + monkeypatch.delenv('KUMO_RUN_GCP_TESTS', raising=False) + else: + monkeypatch.setenv('KUMO_RUN_GCP_TESTS', run_gcp) + monkeypatch.setenv(bigquery_connector.ENV_GOOGLE_CREDENTIALS_JSON, + '/unused/mock-adc.json') + setup = mock.Mock() + monkeypatch.setattr(testing_bigquery, 'setup_bigquery_credentials', setup) + body = mock.Mock(return_value='ran') + + assert testing_bigquery.withBigQueryCredentials(body)() == 'ran' + setup.assert_called_once_with(None) + body.assert_called_once_with() From de91b1ecf01e377d47393a3f970530c30e1026b3 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Wed, 4 Nov 2026 11:35:01 -0600 Subject: [PATCH 2/4] Classify copied epserde source in header policy --- .../epserde/tests/test_review_regressions.rs | 1 + scripts/license_header_policy.json | 252 ++++++++++++++++++ test/scripts/test_check_license_headers.py | 24 ++ 3 files changed, 277 insertions(+) diff --git a/packages/diskgraph/third_party/epserde/tests/test_review_regressions.rs b/packages/diskgraph/third_party/epserde/tests/test_review_regressions.rs index f50d0dfa6..2cdb6d8db 100644 --- a/packages/diskgraph/third_party/epserde/tests/test_review_regressions.rs +++ b/packages/diskgraph/third_party/epserde/tests/test_review_regressions.rs @@ -1,4 +1,5 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. // SPDX-License-Identifier: Apache-2.0 use epserde::prelude::*; use std::ops::ControlFlow; diff --git a/scripts/license_header_policy.json b/scripts/license_header_policy.json index e40c0d7f4..cdbbcc80f 100644 --- a/scripts/license_header_policy.json +++ b/scripts/license_header_policy.json @@ -215,6 +215,258 @@ "kind": "vendored", "license": "MIT", "reason": "Vendored upstream PtrHash sources; preserve the upstream files and license." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/Cargo.toml", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/build.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/deser/helpers.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/deser/mem_case.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/deser/mod.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/deser/read.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/deser/reader_with_pos.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/deser/slice_with_pos.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/array.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/boxed_slice.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/iter.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/mod.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/prim.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/slice.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/stdlib.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/string.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/tuple.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/vec.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/lib.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/ser/helpers.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/ser/mod.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/ser/write.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/ser/write_with_names.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/traits/copy_type.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/traits/mod.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/traits/type_info.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/utils/aligned_cursor.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/utils/mod.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_bad_deser.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_bad_ser.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_generics.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_max_size_of.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_memcase.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_phantom.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_prim.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_slice.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_std.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_stdlib.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_tuples.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_type_hash_val.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_types.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_zero.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." } ] } diff --git a/test/scripts/test_check_license_headers.py b/test/scripts/test_check_license_headers.py index 46ff5e561..1bf7f2ec0 100644 --- a/test/scripts/test_check_license_headers.py +++ b/test/scripts/test_check_license_headers.py @@ -297,3 +297,27 @@ def test_repository_candidate_tree_passes() -> None: ) assert errors == [] assert scanned > 3_000 + + +def test_epserde_vendor_boundary_is_exact() -> None: + prefix = "packages/diskgraph/third_party/epserde/" + exclusions = [entry for entry in POLICY["exclusions"] + if entry["pattern"].startswith(prefix)] + assert len(exclusions) == 42 + assert all(not any(marker in entry["pattern"] for marker in "*?[") + for entry in exclusions) + for relative in ("Cargo.toml", "src/deser/read.rs", "tests/test_types.rs"): + exclusion = check_license_headers.matching_exclusion( + prefix + relative, POLICY) + assert exclusion is not None + assert exclusion["kind"] == "vendored" + assert exclusion["license"] == "Apache-2.0 OR LGPL-2.1-or-later" + for relative in ("src/utils/alignment.rs", + "tests/test_review_regressions.rs", "new_source.rs"): + assert check_license_headers.matching_exclusion( + prefix + relative, POLICY) is None + assert check_license_headers.validate_text( + prefix + relative, SLASH_HEADER + "fn checked() {}\n", POLICY + ) is None + assert check_license_headers.validate_text( + prefix + relative, "fn unchecked() {}\n", POLICY) is not None From 05c8f0fe53d57f01d12c76466e1b724a7cc5bf17 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Wed, 4 Nov 2026 11:38:59 -0600 Subject: [PATCH 3/4] Make snapshot range checksums portable --- .../diskgraph-ingest/src/snapshot_mirror.rs | 118 +++++++++++++++++- 1 file changed, 115 insertions(+), 3 deletions(-) diff --git a/packages/diskgraph/diskgraph-ingest/src/snapshot_mirror.rs b/packages/diskgraph/diskgraph-ingest/src/snapshot_mirror.rs index 4e90e9704..7348bb414 100644 --- a/packages/diskgraph/diskgraph-ingest/src/snapshot_mirror.rs +++ b/packages/diskgraph/diskgraph-ingest/src/snapshot_mirror.rs @@ -597,7 +597,7 @@ pub fn snapshot_range_checksums( range_bytes: u64, ) -> std::result::Result>, CorruptionError> { use rayon::prelude::*; - use std::os::unix::fs::FileExt; + use std::io::{Read, Seek, SeekFrom}; const CHUNK: u64 = 1 << 20; if range_bytes == 0 { @@ -631,15 +631,18 @@ pub fn snapshot_range_checksums( path: path.clone(), detail: e.to_string(), }; - let file = std::fs::File::open(&path).map_err(io)?; + let mut file = std::fs::File::open(&path).map_err(io)?; let start = range * range_bytes; let end = entry.size_bytes.min(start + range_bytes); let mut hasher = crc32fast::Hasher::new(); let mut buffer = vec![0u8; CHUNK.min(end - start) as usize]; + if start < end { + file.seek(SeekFrom::Start(start)).map_err(io)?; + } let mut offset = start; while offset < end { let length = CHUNK.min(end - offset) as usize; - file.read_exact_at(&mut buffer[..length], offset).map_err(io)?; + file.read_exact(&mut buffer[..length]).map_err(io)?; hasher.update(&buffer[..length]); offset += length as u64; } @@ -1033,4 +1036,113 @@ mod tests { CorruptionError::Io { .. } )); } + + fn crc32_reference(bytes: &[u8]) -> u32 { + let mut crc = !0u32; + for &byte in bytes { + crc ^= u32::from(byte); + for _ in 0..8 { + crc = (crc >> 1) ^ if crc & 1 == 1 { 0xedb8_8320 } else { 0 }; + } + } + !crc + } + + #[test] + fn checksums_cover_parallel_ranges() { + let tmp = tempfile::tempdir().unwrap(); + let span: usize = (1 << 20) + 37; + let first: Vec = (0..(span * 3 + 51)) + .map(|i| (i.wrapping_mul(73) ^ (i >> 7) ^ 17) as u8) + .collect(); + let second: Vec = (0..(span * 2 + 173)) + .map(|i| (i.wrapping_mul(19) ^ (i >> 5) ^ 211) as u8) + .collect(); + write_marker_files( + tmp.path(), + &[("z.bin", &first), ("a.bin", &second), ("empty", b"")], + ); + let manifest = serde_json::to_vec(&Manifest { + schema_version: MANIFEST_SCHEMA_VERSION, + files: vec![ + ManifestEntry { + rel_path: "z.bin".into(), + size_bytes: first.len() as u64, + }, + ManifestEntry { + rel_path: "empty".into(), + size_bytes: 0, + }, + ManifestEntry { + rel_path: "a.bin".into(), + size_bytes: second.len() as u64, + }, + ], + }) + .unwrap(); + std::fs::write(tmp.path().join(MANIFEST_FILENAME), &manifest).unwrap(); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .unwrap(); + let actual = pool + .install(|| snapshot_range_checksums(tmp.path(), span as u64)) + .unwrap(); + assert_eq!(actual.len(), 4); + for (name, bytes) in [ + ("z.bin", first.as_slice()), + ("a.bin", second.as_slice()), + (MANIFEST_FILENAME, manifest.as_slice()), + ] { + assert_eq!( + actual[name], + bytes.chunks(span).map(crc32_reference).collect::>(), + "checksum order differs for {name}" + ); + } + assert_eq!(actual["empty"], vec![0]); + } + + #[test] + fn checksums_reject_invalid_inputs() { + let tmp = tempfile::tempdir().unwrap(); + assert!(matches!( + snapshot_range_checksums(tmp.path(), 0).unwrap_err(), + CorruptionError::Io { path, detail } + if path == tmp.path() && detail == "range_bytes must be positive" + )); + assert!(matches!( + snapshot_range_checksums(tmp.path(), 16).unwrap_err(), + CorruptionError::MissingManifest(path) + if path == tmp.path().join(MANIFEST_FILENAME) + )); + write_manifest( + tmp.path(), + vec![ManifestEntry { + rel_path: "missing.bin".into(), + size_bytes: 0, + }], + ); + assert!(matches!( + snapshot_range_checksums(tmp.path(), 16).unwrap_err(), + CorruptionError::Io { path, .. } if path == tmp.path().join("missing.bin") + )); + } + + #[test] + fn checksums_reject_short_files() { + let tmp = tempfile::tempdir().unwrap(); + write_marker_files(tmp.path(), &[("short.bin", b"short")]); + write_manifest( + tmp.path(), + vec![ManifestEntry { + rel_path: "short.bin".into(), + size_bytes: (1 << 20) + 91, + }], + ); + assert!(matches!( + snapshot_range_checksums(tmp.path(), (1 << 20) + 37).unwrap_err(), + CorruptionError::Io { path, .. } if path == tmp.path().join("short.bin") + )); + } } From cf81f2c5a31eec8711818d2de39bee3acef28d05 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Wed, 4 Nov 2026 12:39:54 -0600 Subject: [PATCH 4/4] Keep BigQuery prerequisite scoped to eligibility --- .../diskgraph-ingest/src/snapshot_mirror.rs | 118 +----------------- 1 file changed, 3 insertions(+), 115 deletions(-) diff --git a/packages/diskgraph/diskgraph-ingest/src/snapshot_mirror.rs b/packages/diskgraph/diskgraph-ingest/src/snapshot_mirror.rs index 7348bb414..4e90e9704 100644 --- a/packages/diskgraph/diskgraph-ingest/src/snapshot_mirror.rs +++ b/packages/diskgraph/diskgraph-ingest/src/snapshot_mirror.rs @@ -597,7 +597,7 @@ pub fn snapshot_range_checksums( range_bytes: u64, ) -> std::result::Result>, CorruptionError> { use rayon::prelude::*; - use std::io::{Read, Seek, SeekFrom}; + use std::os::unix::fs::FileExt; const CHUNK: u64 = 1 << 20; if range_bytes == 0 { @@ -631,18 +631,15 @@ pub fn snapshot_range_checksums( path: path.clone(), detail: e.to_string(), }; - let mut file = std::fs::File::open(&path).map_err(io)?; + let file = std::fs::File::open(&path).map_err(io)?; let start = range * range_bytes; let end = entry.size_bytes.min(start + range_bytes); let mut hasher = crc32fast::Hasher::new(); let mut buffer = vec![0u8; CHUNK.min(end - start) as usize]; - if start < end { - file.seek(SeekFrom::Start(start)).map_err(io)?; - } let mut offset = start; while offset < end { let length = CHUNK.min(end - offset) as usize; - file.read_exact(&mut buffer[..length]).map_err(io)?; + file.read_exact_at(&mut buffer[..length], offset).map_err(io)?; hasher.update(&buffer[..length]); offset += length as u64; } @@ -1036,113 +1033,4 @@ mod tests { CorruptionError::Io { .. } )); } - - fn crc32_reference(bytes: &[u8]) -> u32 { - let mut crc = !0u32; - for &byte in bytes { - crc ^= u32::from(byte); - for _ in 0..8 { - crc = (crc >> 1) ^ if crc & 1 == 1 { 0xedb8_8320 } else { 0 }; - } - } - !crc - } - - #[test] - fn checksums_cover_parallel_ranges() { - let tmp = tempfile::tempdir().unwrap(); - let span: usize = (1 << 20) + 37; - let first: Vec = (0..(span * 3 + 51)) - .map(|i| (i.wrapping_mul(73) ^ (i >> 7) ^ 17) as u8) - .collect(); - let second: Vec = (0..(span * 2 + 173)) - .map(|i| (i.wrapping_mul(19) ^ (i >> 5) ^ 211) as u8) - .collect(); - write_marker_files( - tmp.path(), - &[("z.bin", &first), ("a.bin", &second), ("empty", b"")], - ); - let manifest = serde_json::to_vec(&Manifest { - schema_version: MANIFEST_SCHEMA_VERSION, - files: vec![ - ManifestEntry { - rel_path: "z.bin".into(), - size_bytes: first.len() as u64, - }, - ManifestEntry { - rel_path: "empty".into(), - size_bytes: 0, - }, - ManifestEntry { - rel_path: "a.bin".into(), - size_bytes: second.len() as u64, - }, - ], - }) - .unwrap(); - std::fs::write(tmp.path().join(MANIFEST_FILENAME), &manifest).unwrap(); - let pool = rayon::ThreadPoolBuilder::new() - .num_threads(4) - .build() - .unwrap(); - let actual = pool - .install(|| snapshot_range_checksums(tmp.path(), span as u64)) - .unwrap(); - assert_eq!(actual.len(), 4); - for (name, bytes) in [ - ("z.bin", first.as_slice()), - ("a.bin", second.as_slice()), - (MANIFEST_FILENAME, manifest.as_slice()), - ] { - assert_eq!( - actual[name], - bytes.chunks(span).map(crc32_reference).collect::>(), - "checksum order differs for {name}" - ); - } - assert_eq!(actual["empty"], vec![0]); - } - - #[test] - fn checksums_reject_invalid_inputs() { - let tmp = tempfile::tempdir().unwrap(); - assert!(matches!( - snapshot_range_checksums(tmp.path(), 0).unwrap_err(), - CorruptionError::Io { path, detail } - if path == tmp.path() && detail == "range_bytes must be positive" - )); - assert!(matches!( - snapshot_range_checksums(tmp.path(), 16).unwrap_err(), - CorruptionError::MissingManifest(path) - if path == tmp.path().join(MANIFEST_FILENAME) - )); - write_manifest( - tmp.path(), - vec![ManifestEntry { - rel_path: "missing.bin".into(), - size_bytes: 0, - }], - ); - assert!(matches!( - snapshot_range_checksums(tmp.path(), 16).unwrap_err(), - CorruptionError::Io { path, .. } if path == tmp.path().join("missing.bin") - )); - } - - #[test] - fn checksums_reject_short_files() { - let tmp = tempfile::tempdir().unwrap(); - write_marker_files(tmp.path(), &[("short.bin", b"short")]); - write_manifest( - tmp.path(), - vec![ManifestEntry { - rel_path: "short.bin".into(), - size_bytes: (1 << 20) + 91, - }], - ); - assert!(matches!( - snapshot_range_checksums(tmp.path(), (1 << 20) + 37).unwrap_err(), - CorruptionError::Io { path, .. } if path == tmp.path().join("short.bin") - )); - } }