diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 8edbd62..100beb6 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -1,41 +1,14 @@ -# Ignore advisories for transitive dependencies we cannot directly upgrade. -# These come from solana-sdk and sqlx crate trees. - +# Narrow upstream exceptions; review when the dependency tree changes. [advisories] ignore = [ - # rsa 0.9.9 (via sqlx-mysql): no fix available upstream. - # We use PostgreSQL, not MySQL -- sqlx-mysql is pulled in by the sqlx - # macros crate unconditionally. + # Optional sqlx-mysql resolution only. `cargo tree -i rsa --target all` + # has no active path: this PostgreSQL-only application does not build RSA. + # No patched rsa 0.9 release is available. "RUSTSEC-2023-0071", - # time 0.3.44 (via solana-*, alloy, x509-parser): stack exhaustion DoS. - # Transitive dependency pinned by solana-sdk 3.x; cannot upgrade independently. - "RUSTSEC-2026-0009", - - # keccak 0.1.5 (via solana-zk-sdk/merlin/sha3): unsound ARMv8 assembly. - # Transitive dependency pinned by solana-zk-sdk; cannot upgrade independently. - "RUSTSEC-2026-0012", - - # rand 0.8.5 (via alloy-signer-local, governor, solana-*): unsound when a - # CUSTOM `log::Log` implementation calls `rand::rng()` / ThreadRng methods - # and triggers a reseed mid-log. Affects rand `>= 0.7, < 0.9.3`; fix is - # rand 0.9.3+, which is a major-version bump the transitive deps above - # have not yet taken. - # - # Not exploitable in this workspace: - # - We do not implement `log::Log`. We use `tracing_subscriber::fmt` (a - # tracing Layer, not a `log::Log` impl). - # - No code in this repo calls `rand::rng()` or `rand::thread_rng()` from - # inside a logger. Transitive crates that use `rand` do so outside of - # logging paths (key generation, jitter, QUIC connection IDs, etc.). - # Revisit when alloy / governor / solana-* bump past rand 0.9.3. - # - # We also run `cargo update -p rand` to keep the rand 0.9.x line on the - # latest 0.9.x (currently 0.9.4, past the advisory cutoff of 0.9.3). - "RUSTSEC-2026-0097", + # alloy-provider 1.7.1 uses lru 0.16.4 with u64 and B256 keys. + # Neither key has a Drop implementation. The advisory requires a key's + # Drop to panic during pop(), followed by catch_unwind and cache reuse. + # Revisit when Alloy adopts lru >=0.18.2; do not add panicking cache keys. + "RUSTSEC-2026-0253", ] - -[yanked] -# keccak 0.1.5 is yanked but pinned transitively by solana-zk-sdk/merlin. -# Cannot be upgraded without a new solana-sdk release. -enabled = false diff --git a/.dockerignore b/.dockerignore index 532e97a..0821687 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,9 @@ .git target/ -node_modules/ +**/node_modules/ +**/__pycache__/ +web-tests/artifacts/ +*.log .smithers/ .claude/ .entire/ @@ -12,3 +15,6 @@ bun.lock *.md .env +spectraplex.toml +exports/ +.local/ diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index cc86323..3511d2b 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -6,10 +6,14 @@ on: pull_request: paths: - "Cargo.lock" + - "**/Cargo.toml" + - ".cargo/audit.toml" push: branches: [main] paths: - "Cargo.lock" + - "**/Cargo.toml" + - ".cargo/audit.toml" permissions: contents: read @@ -21,6 +25,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - name: Verify unused-RSA exception remains valid + run: | + tree=$(cargo tree --locked -i rsa --target all) + test -z "$tree" - uses: rustsec/audit-check@v2.0.0 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4d8e2b..447386c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,20 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - - run: cargo test --workspace + - run: cargo test --locked --workspace + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: web-tests/package-lock.json + - run: npm ci --prefix web-tests + - run: cd web-tests && npx playwright install --with-deps chromium + - run: ./scripts/smoke-test.sh --browser + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: browser-failure-artifacts + path: web-tests/artifacts build: name: Build (release) @@ -75,4 +88,32 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - - run: cargo build --release --workspace + - run: cargo build --locked --release --workspace + + docker: + name: Docker first run + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - name: Build and boot + run: | + ./scripts/setup.sh + docker compose up --build -d --wait --wait-timeout 120 + curl -fsS http://127.0.0.1:3000/ready + test "$(docker compose exec -T api id -u)" != 0 + - name: Check pipeline and persistence + run: | + set -a; source .env; set +a + export SPECTRAPLEX_TEST_URL=http://127.0.0.1:3000 + export SPECTRAPLEX_TEST_KEY="$SPECTRAPLEX_API_KEY" + python3 scripts/container_test.py prepare .local/container-check.json + docker compose down + docker compose up -d --wait --wait-timeout 120 + python3 scripts/container_test.py verify .local/container-check.json + - name: Container logs + if: failure() + run: docker compose logs --tail 100 + - name: Stop + if: always() + run: docker compose down -v diff --git a/.gitignore b/.gitignore index f638c70..9a4e8a2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,20 @@ /target +/.env +/spectraplex.toml +/exports/ +/.local/ n*.md p*.md docs/ design/ /workflow.tsx /.smithers/ -node_modules/* +node_modules/ +__pycache__/ +/web-tests/artifacts/ .claude/* .entire/* AGENTS.md CLAUDE.md bun.lock -package.json +/package.json diff --git a/Cargo.lock b/Cargo.lock index c3ef088..76d200c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -175,7 +175,7 @@ dependencies = [ "either", "k256", "once_cell", - "rand 0.8.5", + "rand 0.8.8", "secp256k1", "serde", "serde_json", @@ -596,7 +596,7 @@ dependencies = [ "alloy-signer", "async-trait", "k256", - "rand 0.8.5", + "rand 0.8.8", "thiserror 2.0.17", ] @@ -786,7 +786,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -797,14 +797,14 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arc-swap" @@ -870,6 +870,23 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ark-ff" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7a806ac6c8307b929df4645776290a50ee2aac754ad09d8bdf73391309e43af" +dependencies = [ + "ark-ff-asm 0.6.0", + "ark-ff-macros 0.6.0", + "ark-serialize 0.6.0", + "ark-std 0.6.0", + "digest 0.10.7", + "educe", + "num-bigint 0.4.6", + "num-traits", + "zeroize", +] + [[package]] name = "ark-ff-asm" version = "0.3.0" @@ -900,6 +917,16 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "ark-ff-asm" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1479009684adc073dff49a1025d3a7065b317a9ead25aaaca38cdc70058ba8a2" +dependencies = [ + "quote", + "syn 2.0.111", +] + [[package]] name = "ark-ff-macros" version = "0.3.0" @@ -938,6 +965,19 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "ark-ff-macros" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0691ed21ef00ef89c1e9bda832eba493dda3ec2f8d892fb25b705f73f06bb8" +dependencies = [ + "num-bigint 0.4.6", + "num-traits", + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "ark-serialize" version = "0.3.0" @@ -971,6 +1011,30 @@ dependencies = [ "num-bigint 0.4.6", ] +[[package]] +name = "ark-serialize" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a74dd304fd536fb95d0a328e72be759209cc496a9da094c5bc56e5fea4f9e86b" +dependencies = [ + "ark-serialize-derive", + "ark-std 0.6.0", + "digest 0.10.7", + "num-bigint 0.4.6", + "serde_with", +] + +[[package]] +name = "ark-serialize-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f153690697a2b91e5e1251ff98411ee5371500a111a0fd317a70e588eb300f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "ark-std" version = "0.3.0" @@ -978,7 +1042,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df2c09229cbc5a028b1d70e00fdb2acee28b1055dfb5ca73eea49c5a25c4e7c" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.8.8", ] [[package]] @@ -988,7 +1052,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.8.8", ] [[package]] @@ -998,7 +1062,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "246a225cc6131e9ee4f24619af0f19d67761fff15d7ccc22e42b80846e69449a" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.8.8", +] + +[[package]] +name = "ark-std" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "367c9c827ed431bff6868b7aa926e05b16eb46603cc8b6e768e4a5553fa1d155" +dependencies = [ + "num-traits", + "rand 0.8.8", ] [[package]] @@ -1640,15 +1714,6 @@ version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "console" version = "0.16.2" @@ -1795,9 +1860,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", ] @@ -1980,11 +2045,10 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.5" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] @@ -2242,7 +2306,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -2258,11 +2322,10 @@ dependencies = [ [[package]] name = "event-listener" -version = "5.4.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -2405,7 +2468,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" dependencies = [ "byteorder", - "rand 0.8.5", + "rand 0.8.8", "rustc-hex", "static_assertions", ] @@ -2666,7 +2729,7 @@ dependencies = [ "parking_lot", "portable-atomic", "quanta", - "rand 0.8.5", + "rand 0.8.8", "smallvec", "spinning_top", ] @@ -2687,7 +2750,7 @@ dependencies = [ "parking_lot", "portable-atomic", "quanta", - "rand 0.8.5", + "rand 0.8.8", "smallvec", "spinning_top", ] @@ -2705,9 +2768,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.12" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -3319,9 +3382,9 @@ dependencies = [ [[package]] name = "keccak" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" dependencies = [ "cpufeatures", ] @@ -3407,9 +3470,9 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lru" -version = "0.16.3" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" dependencies = [ "hashbrown 0.16.1", ] @@ -3580,7 +3643,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3629,7 +3692,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.5", + "rand 0.8.8", "smallvec", "zeroize", ] @@ -3646,9 +3709,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-derive" @@ -4185,7 +4248,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1" dependencies = [ "heck", - "itertools 0.13.0", + "itertools 0.12.1", "log", "multimap", "once_cell", @@ -4207,7 +4270,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.111", @@ -4303,9 +4366,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "bytes", "fastbloom", @@ -4335,7 +4398,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -4361,9 +4424,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -4654,14 +4717,15 @@ dependencies = [ [[package]] name = "ruint" -version = "1.17.2" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c141e807189ad38a07276942c6623032d3753c8859c146104ac2e4d68865945a" +checksum = "2973657b5127d510e230f5c63d2d106af9c8f79393d8b9f4647323e8196bdde5" dependencies = [ "alloy-rlp", "ark-ff 0.3.0", "ark-ff 0.4.2", "ark-ff 0.5.0", + "ark-ff 0.6.0", "bytes", "fastrlp 0.3.1", "fastrlp 0.4.0", @@ -4671,7 +4735,7 @@ dependencies = [ "parity-scale-codec", "primitive-types", "proptest", - "rand 0.8.5", + "rand 0.8.8", "rand 0.9.4", "rlp", "ruint-macro", @@ -4741,14 +4805,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.23.35" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "log", "once_cell", @@ -4799,7 +4863,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -4810,9 +4874,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "ring", "rustls-pki-types", @@ -4913,7 +4977,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50c5943d326858130af85e049f2661ba3c78b26589b8ab98e65e80ae44a1252" dependencies = [ "bitcoin_hashes", - "rand 0.8.5", + "rand 0.8.8", "secp256k1-sys", "serde", ] @@ -5343,7 +5407,7 @@ dependencies = [ "curve25519-dalek", "five8 1.0.0", "five8_const", - "rand 0.8.5", + "rand 0.8.8", "serde", "serde_derive", "solana-atomic-u64", @@ -5541,7 +5605,7 @@ dependencies = [ "futures-util", "indexmap 2.12.1", "log", - "rand 0.8.5", + "rand 0.8.8", "rayon", "solana-keypair", "solana-measure", @@ -5822,7 +5886,7 @@ dependencies = [ "ed25519-dalek", "ed25519-dalek-bip32", "five8 1.0.0", - "rand 0.8.5", + "rand 0.8.8", "solana-address 2.0.0", "solana-derivation-path", "solana-seed-derivable", @@ -5944,7 +6008,7 @@ dependencies = [ "itertools 0.12.1", "log", "nix", - "rand 0.8.5", + "rand 0.8.8", "serde", "socket2", "solana-serde", @@ -6014,7 +6078,7 @@ dependencies = [ "libc", "log", "nix", - "rand 0.8.5", + "rand 0.8.8", "rayon", "serde", "solana-hash 3.1.0", @@ -6141,7 +6205,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8909d399deb0851aa524420beeb5646b115fd253ef446e35fe4504c904da3941" dependencies = [ - "rand 0.8.5", + "rand 0.8.8", "solana-address 1.1.0", ] @@ -6535,7 +6599,7 @@ checksum = "4bb8057cc0e9f7b5e89883d49de6f407df655bb6f3a71d0b7baf9986a2218fd9" dependencies = [ "ed25519-dalek", "five8 0.2.1", - "rand 0.8.5", + "rand 0.8.8", "serde", "serde-big-array", "serde_derive", @@ -6632,7 +6696,7 @@ dependencies = [ "percentage", "quinn", "quinn-proto", - "rand 0.8.5", + "rand 0.8.8", "rustls", "smallvec", "socket2", @@ -6668,7 +6732,7 @@ version = "3.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a3f2dab6bc6fd16f43942922ba53d7bc947666ef3a121d74ccb426f01ae1eaf" dependencies = [ - "rand 0.8.5", + "rand 0.8.8", ] [[package]] @@ -6843,7 +6907,7 @@ dependencies = [ "base64 0.22.1", "bincode", "log", - "rand 0.8.5", + "rand 0.8.8", "solana-packet", "solana-perf", "solana-short-vec", @@ -6940,7 +7004,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce24ba7de24ea7caec6d811dfbc9eb234f3aee8aaa748f5dd7fc4b1fb3002aaa" dependencies = [ "agave-feature-set", - "rand 0.8.5", + "rand 0.8.8", "semver 1.0.27", "serde", "solana-sanitize", @@ -6991,7 +7055,7 @@ dependencies = [ "merlin", "num-derive", "num-traits", - "rand 0.8.5", + "rand 0.8.8", "serde", "serde_derive", "serde_json", @@ -7012,7 +7076,7 @@ dependencies = [ [[package]] name = "spectraplex-adapters" -version = "0.1.0" +version = "1.0.0" dependencies = [ "alloy", "anyhow", @@ -7043,7 +7107,7 @@ dependencies = [ [[package]] name = "spectraplex-api" -version = "0.1.0" +version = "1.0.0" dependencies = [ "anyhow", "async-trait", @@ -7072,7 +7136,7 @@ dependencies = [ [[package]] name = "spectraplex-cli" -version = "0.1.0" +version = "1.0.0" dependencies = [ "anyhow", "chrono", @@ -7091,7 +7155,7 @@ dependencies = [ [[package]] name = "spectraplex-core" -version = "0.1.0" +version = "1.0.0" dependencies = [ "anyhow", "async-trait", @@ -7110,9 +7174,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" dependencies = [ "lock_api", ] @@ -7477,7 +7541,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rand 0.8.5", + "rand 0.8.8", "rsa", "serde", "sha1", @@ -7519,7 +7583,7 @@ dependencies = [ "memchr", "num-bigint 0.4.6", "once_cell", - "rand 0.8.5", + "rand 0.8.8", "serde", "serde_json", "sha2", @@ -7739,7 +7803,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -7802,30 +7866,29 @@ dependencies = [ [[package]] name = "time" -version = "0.3.44" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.24" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -8252,7 +8315,7 @@ dependencies = [ "httparse", "log", "native-tls", - "rand 0.8.5", + "rand 0.8.8", "sha1", "thiserror 1.0.69", "utf-8", diff --git a/Cargo.toml b/Cargo.toml index bf8c274..e94afa6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,4 +5,17 @@ members = [ "cli", "api", ] -resolver = "2" \ No newline at end of file +resolver = "2" + +[workspace.package] +version = "1.0.0" +edition = "2021" +license = "MIT" +rust-version = "1.94" +repository = "https://github.com/user1303836/spectraplex" + +[profile.dev] +debug = 0 + +[profile.test] +debug = 0 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index b368725..39a67e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,26 +1,24 @@ -FROM rust:1.84-bookworm AS builder - +# syntax=docker/dockerfile:1 +FROM rust:1.94-bookworm AS builder WORKDIR /app COPY . . - -RUN cargo build --release --workspace +ARG CARGO_BUILD_JOBS=2 +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/app/target \ + cargo build --locked --release --workspace \ + && cp target/release/spectraplex-api target/release/spectraplex-cli /tmp/ FROM debian:bookworm-slim - -RUN apt-get update && apt-get install -y ca-certificates libssl3 && rm -rf /var/lib/apt/lists/* - -RUN adduser --disabled-password --gecos "" appuser - -COPY --from=builder /app/target/release/spectraplex-api /usr/local/bin/ -COPY --from=builder /app/target/release/spectraplex-cli /usr/local/bin/ -COPY --from=builder /app/migrations /app/migrations - -RUN chown -R appuser:appuser /app - +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates libssl3 curl \ + && rm -rf /var/lib/apt/lists/* \ + && adduser --disabled-password --gecos "" appuser \ + && mkdir -p /app/exports && chown -R appuser:appuser /app \ + && chmod 2770 /app/exports +COPY --from=builder /tmp/spectraplex-api /usr/local/bin/ +COPY --from=builder /tmp/spectraplex-cli /usr/local/bin/ WORKDIR /app - USER appuser - EXPOSE 3000 - +HEALTHCHECK --interval=15s --timeout=5s --start-period=30s \ + CMD curl -fsS http://127.0.0.1:3000/ready || exit 1 CMD ["spectraplex-api"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1537316 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Spectraplex contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 518b03d..4a50ede 100644 --- a/README.md +++ b/README.md @@ -3,416 +3,131 @@ [![CI](https://github.com/user1303836/spectraplex/actions/workflows/ci.yml/badge.svg)](https://github.com/user1303836/spectraplex/actions/workflows/ci.yml) [![Security Audit](https://github.com/user1303836/spectraplex/actions/workflows/audit.yml/badge.svg)](https://github.com/user1303836/spectraplex/actions/workflows/audit.yml) -Multi-chain blockchain indexer and ETL pipeline. Ingest raw data from Solana, EVM chains, and Hyperliquid, normalize it into queryable datasets, and export it as CSV or JSONL. +**Your self-hosted blockchain data workbench.** Explore wallet activity across Solana, Ethereum-family networks and Hyperliquid. Import trusted data, inspect transactions, and download CSV or JSONL—with provenance. Browser UI, REST API and CLI; no telemetry or frontend build required. -## Quick Start +## Start here + +Requires Docker with Compose v2 (Windows: use WSL). Allow roughly 4 GB RAM and 10 GB free disk for the first build, which takes several minutes. ```bash -# Clone and build git clone https://github.com/user1303836/spectraplex.git cd spectraplex -cargo build --workspace - -# Start PostgreSQL -./scripts/local-dev.sh - -# Start the API server (auto-runs migrations on first start) -cargo run --bin spectraplex-api -# → http://127.0.0.1:3000/health - -# Run the smoke test (in another terminal) -./scripts/smoke-test.sh - -# Or skip provider-dependent ingestion if you don't have live RPC access: -# ./scripts/smoke-test.sh --skip-ingest -``` - -Requires Rust (stable) and PostgreSQL 15+. Docker handles Postgres if you don't have one running. - -## Features - -**Ingest from supported wallet and target paths through the API.** Wallet ingest is the stable MVP path; selected non-wallet targets are available only where the support matrix below labels them supported. - -**Bronze-native Silver materialization.** Raw data is extracted directly into canonical Silver datasets — token transfers, balance deltas, decoded events, fills, funding, and positions — without V1 compatibility overhead. - -**Durable control plane.** Ingestion jobs, stream subscriptions, export jobs, and materialization runs are all Postgres-backed with worker loops, heartbeats, lease-based ownership, and restart recovery. - -**Real-time streaming with full lineage.** Solana gRPC and Hyperliquid WebSocket streams create ingestion runs, persist V2 checkpoints, and auto-trigger downstream materialization — the same lineage model as backfill. - -**Query and export structured datasets.** Silver and Gold datasets are available through a REST API with filtering, pagination, and async export to CSV, JSONL, local files, or webhooks. - -**V2-backed wallet reads.** Legacy wallet endpoints (`/v1/transactions/:wallet`, `/v1/ledger/:wallet`, etc.) are served from V2 tables behind unchanged response shapes. - -**Built-in analytics endpoints.** Trader PnL, market stats, protocol activity, and TVL queries ship out of the box. - -## Architecture - -Data flows through three tiers: - +./scripts/setup.sh +docker compose up --build -d --wait ``` -Ingestion → Bronze (raw_transactions) → Silver (token_transfers, etc.) → Gold (wallet_ledger, etc.) - ↓ ↓ - target_matches dataset_completeness - ingestion_runs dataset_versions - V2 checkpoints dataset_watermarks -``` - -- **Bronze** is canonical and target-agnostic. Raw chain data lives in `raw_transactions` and is linked to `ingestion_runs` and `target_matches`. -- **Silver** datasets are extracted directly from Bronze using chain-specific parsers. Every Silver row carries `raw_transaction_id` for provenance. -- **Gold** datasets are derived from Silver for wallet, tax, forensics, and analytics use cases. - -All operational state — jobs, streams, exports, materialization runs — is durable in PostgreSQL. No runtime truth lives in in-memory maps. - -## Supported Chains - -| Chain | Data Source | Networks | -|-------|------------|----------| -| **Solana** | RPC + Yellowstone gRPC | `solana-mainnet` | -| **Ethereum** | `eth_getLogs` via alloy | `ethereum-mainnet` | -| **Base** | `eth_getLogs` via alloy | `base-mainnet` | -| **Arbitrum** | `eth_getLogs` via alloy | `arbitrum-mainnet` | -| **HyperEVM** | `eth_getLogs` via alloy | `hyperevm-mainnet` | -| **Hyperliquid** | REST + WebSocket | `hypercore-mainnet` | - -## Datasets -| Dataset | Tier | Description | -|---------|------|-------------| -| `token_transfers` | Silver | Token transfer records across all chains | -| `native_balance_deltas` | Silver | Native currency balance changes per account | -| `decoded_events` | Silver | ABI-decoded EVM events and Solana logs | -| `hl_fills` | Silver | Hyperliquid trade fills | -| `hl_funding` | Silver | Hyperliquid funding payments | -| `positions` | Silver | Position state changes from fills and liquidations | -| `wallet_ledger` | Gold | Ledger entries with counterparty tracking | -| `balance_history` | Gold | Per-asset balance snapshots over time | -| `hl_pnl_summary` | Gold | Hyperliquid PnL per coin per period | -| `hl_trade_history` | Gold | Hyperliquid trades with entry/exit grouping | -| `protocol_events` | Gold | Protocol events derived from decoded logs | -| `pool_snapshots` | Gold | Pool state snapshots from events and transfers | +Open **http://localhost:3000**. Connect using the value of `SPECTRAPLEX_API_KEY` in `.env`. -## MVP Support Matrix +1. Click **Solana**, **Ethereum** or **Hyperliquid** under sample data. No RPC account or funds needed. +2. Choose a dataset, filter dates, or open **View JSON**. Normalized results appear when the background job finishes. +3. Click **Export CSV** or **Export JSONL**. Download the file and its provenance from **Activity**. -Support levels: +Samples are synthetic and isolated from real networks. Reloading them is safe. Keys stay in browser memory; reloading disconnects you. -- **Stable** — intended for the documented MVP path and covered by local checks or deterministic tests. -- **Beta** — usable but still needs broader correctness fixtures or provider hardening before production-grade claims. -- **Experimental** — implemented building blocks exist, but operators should expect sharp edges. -- **Unsupported** — intentionally not advertised as working yet. +**Real data:** set your RPC URLs in `spectraplex.toml`, run `docker compose restart api`, then **Add a target → Fetch activity**. Public RPCs may throttle requests. Solana's **Fetch older** walks backwards through history. EVM targets accept an optional starting block and scan forwards in bounded batches; the default is recent blocks. `ingest_limit` controls each batch (EVM: blocks; Solana: transactions). -### Target and Ingestion Support +**Sharing:** create tenant keys in **API keys**, not copies of your admin key. Admin-created keys start isolated workspaces; tenant-created keys share that tenant's workspace. Administrators can access all data. -| Target / path | Support | Notes | -|---------------|---------|-------| -| Solana wallet via `POST /v1/ingest` or `POST /v1/targets/:id/ingest` | Stable | V2-backed wallet ingest remains the primary stable path for wallet ledger, balance, and export workflows. | -| EVM wallet via `POST /v1/ingest` or `POST /v1/targets/:id/ingest` | Beta | Wallet ingest exists, but chain-id/finality/reorg hardening must land before production-grade support claims. | -| Hyperliquid wallet via `POST /v1/ingest` or streams | Beta | REST/WebSocket flows exist, but provider-rate-limit/backoff and gap-detection hardening are still tracked separately. | -| Hyperliquid market target via `POST /v1/targets/:id/ingest` | Beta | First non-wallet target-centric connector path. This path currently writes Bronze market data; Silver/Gold auto-materialization is not yet part of the market-target contract. | -| EVM contract/topic target via target-centric ingest | Unsupported | Connector/runtime hardening is intentionally deferred. Do not treat EVM contract/topic targets as a supported runtime path yet. | -| CLI `normalize` | Legacy compatibility | Available for offline/file-based workflows, but the API ingestion + auto-materialization path is the recommended V2 flow. | +**Operations:** `/ready` checks the database. `docker compose logs --tail 100 api` shows diagnostics. PostgreSQL and exports persist in named volumes; back up both. `docker compose down -v` deletes them. If ports are occupied, set `SPECTRAPLEX_PORT` or `POSTGRES_PORT` in `.env`. Keep the local credentials private; use TLS and hardened database credentials before exposing the service remotely. -### Dataset and Export Support +## What you can work with -| Dataset flow | Support | Notes | -|--------------|---------|-------| -| Silver dataset query/export with `target_id` for materialized wallet paths | Stable | Silver tables carry raw lineage and target matching. Tenant requests must include an owned `target_id`. Non-wallet targets are only query/export-supported where their row above says materialization is part of the contract. | -| `wallet_ledger` Gold query/export | Stable | Target-scoped wallet ledger reads use Gold-specific filtering and include export provenance. | -| `balance_history` Gold query/export | Beta | Materialized with DB-seeded running balances; keep representative correctness fixtures current before production-grade use. | -| Hyperliquid Gold datasets (`hl_pnl_summary`, `hl_trade_history`) | Beta | Durable Gold tables exist; correctness depends on fill/funding semantics and provider completeness. | -| Protocol/TVL Gold datasets (`protocol_events`, `pool_snapshots`) | Experimental | Query/export paths exist, but protocol semantics and TVL calculations need representative fixtures before stable claims. | -| Dataset completeness/status | Stable for owned targets | Tenant-scoped API keys see owner-filtered target completeness/status; dataset version metadata remains global. | -| Export provenance sidecars | Stable | Dataset exports include version and completeness metadata when available. | +| Input | Useful datasets | +|---|---| +| Solana wallets, including v0 transactions | Raw transactions, token transfers, native balance deltas, wallet ledger, indexed balance history | +| EVM wallets | Raw transactions, ERC20 transfers, top-level native value and execution gas, wallet ledger, indexed balance history | +| Hyperliquid wallets | Fills, funding, deposits/withdrawals, wallet ledger; beta PnL and trade history | +| Hyperliquid markets; EVM contracts/topic filters via API | Raw transaction query/export | +| Trusted JSON/JSONL imports (admin only) | The same durable ingestion → normalization → export pipeline | -Known high/critical provider and operational hardening issues are tracked in GitHub issues #248-#267. The support levels above are intentionally conservative until those risks are either fixed or scoped out of a stable path. +The dataset selector and `GET /v1/datasets` list available datasets. Configure additional networks/providers using `spectraplex.toml.example`. -## Supported Path vs Legacy Compatibility +**Know the limits:** +- This is bounded indexing, not a complete-chain index or verified account balance service. EVM scans stop 12 blocks behind the head; that is not a reorg/finality guarantee. Hyperliquid providers may retain only limited history. +- A “complete” dataset status means its indexed batch was processed, not that all historical activity was found. Inspect coverage and export provenance. +- EVM wallet accounting excludes internal transfers, NFTs, and fees beyond execution gas. Unknown ERC20 decimals are `-1`: amounts remain raw units and are excluded from financial totals. +- Balances are cumulative changes in indexed events. PnL/trade grouping is beta; protocol/TVL analytics and live streaming are experimental. Exports are not tax returns. +- Imports are trusted, not independently chain-verified. Existing Bronze payloads are immutable. Older arrivals can rebuild known balance deltas; legacy snapshots without deltas fail closed rather than invent history. -Spectraplex has two operational paths: +## API: sample → query → download -**Supported MVP path (recommended):** -1. Register a target via API (`POST /v1/targets`) or use the wallet ingest shortcut -2. Trigger ingestion via API (`POST /v1/targets/:id/ingest` or `POST /v1/ingest` for wallet shortcuts) -3. Workers write V2 Bronze (`raw_transactions`); stable wallet paths auto-materialize Silver/Gold -4. Query or export from durable V2 tables with an owned `target_id` - -This path is fully tenant-scoped, survives restarts, and produces completeness/provenance metadata for every export. - -**Legacy compatibility path (CLI `normalize`):** -- `spectraplex-cli normalize` parses V1 transactions and writes `ledger_entries` directly -- API workers optionally write V1 tables (`transactions`, `indexer_checkpoints`) for backward compatibility -- Controlled by `enable_v1_compat_writes` config (default: `true`) - -Operators who no longer need V1 tables can set `enable_v1_compat_writes = false` in `spectraplex.toml`. The CLI `normalize` command remains available for offline/file-based workflows but is not the recommended pipeline. - -## API - -All `/v1/*` routes require `Authorization: Bearer `. - -Tenant-scoped API keys (created via `POST /v1/api-keys`) are required for tenant isolation on dataset queries, dataset status/completeness, and exports. The legacy config key (`SPECTRAPLEX_API_KEY`) can also call these endpoints (without tenant isolation) and is used to bootstrap the first tenant key. - -### Ingestion and Jobs - -```bash -# Trigger wallet ingestion (wallet + network; API auto-creates the target) -curl -X POST http://127.0.0.1:3000/v1/ingest \ - -H Authorization:\ Bearer\ *** \ - -H Content-Type:\ application/json \ - -d '{"wallet": "
", "network": "solana-mainnet"}' - -# Trigger target-centric ingestion for an existing target -curl -X POST http://127.0.0.1:3000/v1/targets//ingest \ - -H Authorization:\ Bearer\ *** \ - -H Content-Type:\ application/json \ - -d '{"mode": "backfill"}' - -# Check job status -curl -H Authorization:\ Bearer\ *** \ - http://127.0.0.1:3000/v1/jobs/ -``` - -Ingestion jobs are durable — they persist across restarts and are claimed by background workers with heartbeat-based lease management. - -### Real-Time Streaming +Requires `curl` and `jq`. Run from the repository after starting the service: ```bash -# Start a Solana gRPC stream (requires SOLANA_GRPC_URL and legacy/admin key) -curl -X POST http://127.0.0.1:3000/v1/stream/start \ - -H "Authorization: Bearer $LEGACY_KEY" \ - -H "Content-Type: application/json" \ - -d '{"chain": "solana"}' - -# Start a Hyperliquid WebSocket stream for a wallet -curl -X POST http://127.0.0.1:3000/v1/stream/start \ - -H "Authorization: Bearer $API_KEY" \ - -H "Content-Type: application/json" \ - -d '{"chain": "hyperliquid", "wallet": "0x..."}' - -# List active streams -curl -H "Authorization: Bearer $API_KEY" \ - http://127.0.0.1:3000/v1/streams - -# Stop a stream -curl -X POST http://127.0.0.1:3000/v1/stream//stop \ - -H "Authorization: Bearer $API_KEY" -``` - -Stream subscriptions are durable and create `ingestion_runs` per flush batch with full Bronze lineage. Hyperliquid streams subscribe to user fills, funding, and ledger updates via WebSocket with automatic reconnection. - -### Targets and Networks - -```bash -# Register an indexing target -curl -X POST http://127.0.0.1:3000/v1/targets \ - -H "Authorization: Bearer $SPECT...KEY" \ - -H "Content-Type: application/json" \ - -d '{"kind": "wallet", "network": "solana-mainnet", "address": "
", "mode": "both"}' - -# List networks -curl -H "Authorization: Bearer $SPECT...KEY" http://127.0.0.1:3000/v1/networks -``` - -### API Keys - -Spectraplex supports two authentication modes: -- **Legacy config key** — set via `SPECTRAPLEX_API_KEY` or `api_key` in config. Full admin access, no tenant isolation. -- **Tenant-scoped keys** — created via the API, stored in Postgres, isolated by owner. +source .env +BASE="http://127.0.0.1:${SPECTRAPLEX_PORT:-3000}" +api() { curl -fsS -H "Authorization: Bearer $SPECTRAPLEX_API_KEY" \ + -H 'Content-Type: application/json' "$@"; } + +# Raw records are available immediately; normalization runs in the background. +SAMPLE=$(api -X POST "$BASE/v1/demo/ethereum" -d '{}') +TARGET=$(printf '%s' "$SAMPLE" | jq -r .target_id) +api "$BASE/v1/datasets/raw_transactions/records?target_id=$TARGET&limit=50" | jq + +EXPORT=$(api -X POST "$BASE/v1/export/dataset" \ + -d "{\"dataset\":\"raw_transactions\",\"target_id\":\"$TARGET\",\"format\":\"jsonl\"}" | jq -r .id) +for attempt in $(seq 1 60); do + STATE=$(api "$BASE/v1/export/jobs/$EXPORT" | jq -r .state) + [[ "$STATE" == completed || "$STATE" == failed ]] && break + sleep 1 +done +api "$BASE/v1/export/jobs/$EXPORT" | jq +if [[ "$STATE" == completed ]]; then + api "$BASE/v1/export/jobs/$EXPORT/download" -o history.jsonl + api "$BASE/v1/export/jobs/$EXPORT/download?provenance=true" -o history.provenance.json +fi +``` + +Use `wallet_ledger`, `token_transfers` or another dataset name for normalized results after materialization completes. Record queries support `target_id`, `network`, `time_start`/`time_end` (Unix seconds), `limit` and `offset`. + +| Action | Route | +|---|---| +| Create/list targets | `POST` / `GET /v1/targets` | +| Fetch a target | `POST /v1/targets/{id}/ingest` with `{"mode":"incremental"}` or `{"mode":"backfill"}` | +| Inspect jobs | `GET /v1/jobs`, `GET /v1/jobs/{id}` | +| Create a tenant key | `POST /v1/api-keys` with `{"name":"Research"}` | +| Revoke a key | `DELETE /v1/api-keys/{id}` | + +### Import your own records + +Create a real-network target, then open **Import raw transactions**. Accepts JSON arrays, `{"records":[…]}`, or JSONL; at most 500 records / 1 MiB per upload. Each record needs `tx_hash`, `timestamp` (Unix seconds), and a chain-specific `raw_metadata` RPC payload. Optional: `block_number`. Download sample payloads from the sidebar to see the shapes; don't present synthetic records as real activity. + +For an existing real target and a trusted `records.jsonl` file, the equivalent API request is: ```bash -# Create a tenant-scoped API key (use legacy key for auth) -curl -X POST http://127.0.0.1:3000/v1/api-keys \ - -H "Authorization: Bearer $SPECT...CONFIG_KEY" \ - -H "Content-Type: application/json" \ - -d '{"name": "production"}' -# → returns {"id":"...","key":"spx_...","owner_id":"..."} - -# List your active keys -curl -H "Authorization: Bearer $SPX...TENANT_KEY" \ - http://127.0.0.1:3000/v1/api-keys - -# Revoke a key -curl -X DELETE http://127.0.0.1:3000/v1/api-keys/ \ - -H "Authorization: Bearer $SPX...TENANT_KEY" +read -r -p 'Real target ID: ' REAL_TARGET +jq -s '{records: .}' records.jsonl | \ + api -X POST "$BASE/v1/targets/$REAL_TARGET/import" --data-binary @- ``` -### Query Datasets - -Tenant-scoped dataset queries require `target_id`: +## CLI and development ```bash -# Query a dataset (tenant-scoped) -curl -H "Authorization: Bearer ***" \ - "http://127.0.0.1:3000/v1/datasets/token_transfers/records?target_id=&network=solana-mainnet&limit=50" - -# Check dataset completeness for your tenant's targets -curl -H "Authorization: Bearer ***" \ - http://127.0.0.1:3000/v1/datasets/token_transfers/completeness - -# Check dataset status for your tenant's targets -curl -H "Authorization: Bearer ***" \ - http://127.0.0.1:3000/v1/datasets/token_transfers/status +docker compose exec api spectraplex-cli --help +docker compose exec api spectraplex-cli list-networks ``` -Dataset completeness is tracked per target and network. After each materialization run, the system upserts a completeness record with coverage bounds (timestamp or block range), record count, and status (`complete`, `partial`, `backfilling`, or `gap`). - -### Export +Prefer the workbench/API for the canonical pipeline. CLI `normalize` is a legacy compatibility path, not the workbench materializer. -Tenant-scoped exports require `target_id`: +To run natively on macOS/Linux: install **Rust 1.94+** and **PostgreSQL 16+** (or start just `docker compose up -d postgres`), run `./scripts/setup.sh`, set `DATABASE_URL` in `.env`, then: ```bash -# Create an export job (CSV or JSONL) -curl -X POST http://127.0.0.1:3000/v1/export/dataset \ - -H "Authorization: Bearer $API_KEY" \ - -H "Content-Type: application/json" \ - -d '{"dataset": "wallet_ledger", "format": "csv", "target_id": "", "network": "solana-mainnet"}' - -# Download when ready -curl -H "Authorization: Bearer $API_KEY" \ - http://127.0.0.1:3000/v1/export/jobs//download -``` - -Export jobs write two files to the configured export directory: -- `{job_id}.csv` or `{job_id}.jsonl` — the data artifact -- `{job_id}.provenance.json` — sidecar with dataset version, completeness status, coverage bounds, and record count - -The provenance file is useful for downstream pipelines that need to know whether the export is complete or partial before processing. - -Export jobs are durable with heartbeat-based worker execution. Supported sinks: `local_file` and `webhook`. - -### Callback HMAC Verification - -When `callback_hmac_secret` is configured, callback payloads include an `X-Spectraplex-Signature` header: - -```bash -curl -X POST http://127.0.0.1:3000/v1/ingest \ - -H "Authorization: Bearer $API_KEY" \ - -H "Content-Type: application/json" \ - -d '{"wallet": "
", "network": "solana-mainnet", "callback_url": "https://your-app.example.com/webhook"}' -``` - -The webhook receiver can verify the signature: - -```python -import hmac -import hashlib - -def verify_signature(payload: bytes, signature_header: str, secret: str) -> bool: - expected = hmac.new(secret.encode(), payload, hashlib.sha256).hexdigest() - return hmac.compare_digest(f"sha256={expected}", signature_header) -``` - -Signature format: `X-Spectraplex-Signature: sha256=`. - -### Analytics - -```bash -# Hyperliquid trader analytics -curl -H "Authorization: Bearer $SPECTRAPLEX_API_KEY" \ - "http://127.0.0.1:3000/v1/analytics/hl/trader?wallet=
" - -# Protocol activity -curl -H "Authorization: Bearer $SPECTRAPLEX_API_KEY" \ - "http://127.0.0.1:3000/v1/analytics/protocol/activity?network=ethereum-mainnet" - -# TVL -curl -H "Authorization: Bearer $SPECTRAPLEX_API_KEY" \ - http://127.0.0.1:3000/v1/analytics/protocol/tvl -``` - -
-Full endpoint reference - -**Ingestion and job control:** -`POST /v1/ingest` | `POST /v1/ingest/batch` | `POST /v1/normalize` | `GET /v1/jobs/:job_id` | `POST /v1/stream/start` | `POST /v1/stream/:stream_id/stop` | `GET /v1/streams` - -**Wallet endpoints (V2-backed):** -`GET /v1/transactions/:wallet` | `GET /v1/transactions/:wallet/:tx_hash` | `GET /v1/ledger/:wallet` | `GET /v1/export/:wallet` | `GET /v1/balances/:wallet` | `GET /v1/stats/:wallet` - -**Targets and networks:** -`POST /v1/targets` | `GET /v1/targets` | `GET /v1/targets/:target_id` | `POST /v1/targets/:target_id/ingest` | `GET /v1/networks` | `GET /v1/networks/:network_id` - -**API keys:** -`POST /v1/api-keys` | `GET /v1/api-keys` | `DELETE /v1/api-keys/:key_id` - -**Datasets:** -`GET /v1/datasets` | `GET /v1/datasets/:name/versions` | `GET /v1/datasets/:name/records` | `GET /v1/datasets/:name/completeness` | `GET /v1/datasets/:name/status` - -**Export:** -`POST /v1/export/dataset` | `GET /v1/export/jobs/:job_id` | `GET /v1/export/jobs/:job_id/download` | `GET /v1/export/tax` - -**Analytics:** -`GET /v1/forensics/activity` | `GET /v1/analytics/hl/trader` | `GET /v1/analytics/hl/market` | `GET /v1/analytics/protocol/activity` | `GET /v1/analytics/protocol/tvl` - -
- -## CLI - -```bash -cargo run --bin spectraplex-cli -- --help -``` - -| Command | Description | -|---------|-------------| -| `init-db` | Create tables, indexes, and seed network data | -| `ingest` | Pull raw transactions from a chain into Bronze storage | -| `normalize` | Materialize Silver datasets from ingested Bronze data | -| `register-target` | Register a wallet, contract, program, or event filter for indexing | -| `list-targets` | List registered targets with optional network/kind filters | -| `list-networks` | Show all seeded networks | - -```bash -# Ingest from any chain -spectraplex-cli --db-url "$DATABASE_URL" ingest --chain ethereum --wallet
--rpc "$EVM_RPC_URL" --limit 10 -spectraplex-cli --db-url "$DATABASE_URL" ingest --chain hyperliquid --wallet
--limit 10 - -# Register a target for ERC-20 Transfer events -spectraplex-cli --db-url "$DATABASE_URL" register-target \ - --kind topic_filter --network ethereum-mainnet \ - --filter-spec '{"topics":["0xddf252ad00000000000000000000000000000000000000000000000000000000"]}' \ - --mode backfill --label "ERC20 transfers" -``` - -## Configuration - -Spectraplex loads config from (in order of precedence): - -1. Defaults -2. `spectraplex.toml` (copy from `spectraplex.toml.example`) -3. `SPECTRAPLEX_*` environment variables -4. Direct env vars (`DATABASE_URL`, `SOLANA_RPC_URL`, etc.) - -Key environment variables: - -```bash -DATABASE_URL=postgresql://localhost/spectraplex -SPECTRAPLEX_API_KEY=*** -SOLANA_RPC_URL=https://api.mainnet-beta.solana.com -EVM_RPC_URL=https://eth.llamarpc.com -# Optional -SOLANA_GRPC_URL=https://your-yellowstone-endpoint -SOLANA_GRPC_TOKEN=*** -# Optional: disable V1 compatibility writes -# SPECTRAPLEX_ENABLE_V1_COMPAT_WRITES=false -``` - -## Project Layout - -``` -spectraplex/ -├── core/ Shared models, config, dataset registry, traits -├── adapters/ Chain adapters, parsers, dual-write layer, repository -├── cli/ CLI entry points -├── api/ Axum HTTP API server + background workers -└── migrations/ PostgreSQL schema and seed data +cargo run --locked --bin spectraplex-api ``` -## Development +Config loads from `spectraplex.toml` (or `SPECTRAPLEX_CONFIG`), then environment overrides. The API applies migrations automatically. Use a **local test database account with CREATEDB permission** for integration tests; PostgreSQL's `psql` and Python 3 must be on PATH: ```bash -cargo fmt --all --check # format check -cargo clippy --workspace --all-targets -- -D warnings # lint -cargo test --workspace # tests +source .env +export TEST_DATABASE_URL="$DATABASE_URL" +cargo fmt --all --check +cargo clippy --locked --workspace --all-targets -- -D warnings +cargo test --locked --workspace +npm ci --prefix web-tests +(cd web-tests && npx playwright install chromium) +./scripts/smoke-test.sh --browser ``` -## License +Node 22+ is only needed for browser tests. The smoke suite uses isolated databases and local provider fixtures, covering three-chain ingestion, financial fixtures, replay, tenant isolation, imports, exports, restart recovery, mobile layout and automated WCAG AA checks. CI also boots containers and checks volume persistence. Reviewed upstream security exceptions live in `.cargo/audit.toml`. -MIT +MIT licensed. Rust workspace: `core/` models, `adapters/` ingestion/storage, `api/` workbench/workers, `cli/` administration, `migrations/` schema. diff --git a/adapters/Cargo.toml b/adapters/Cargo.toml index 54a124c..7ae7b93 100644 --- a/adapters/Cargo.toml +++ b/adapters/Cargo.toml @@ -1,7 +1,10 @@ [package] name = "spectraplex-adapters" -version = "0.1.0" -edition = "2021" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true [dependencies] spectraplex-core = { path = "../core" } diff --git a/adapters/src/dual_write.rs b/adapters/src/dual_write.rs index 483ace7..9681dd8 100644 --- a/adapters/src/dual_write.rs +++ b/adapters/src/dual_write.rs @@ -1364,6 +1364,17 @@ impl Repository { coverage_end: raw_txs.iter().map(|r| r.timestamp).max(), ..Default::default() }; + for raw in raw_txs { + let family = + DatasetRegistry::chain_family_for_network(&raw.network).unwrap_or(ChainFamily::Evm); + if let Err(error) = crate::validate_import_payload(family, &raw.raw_metadata) { + result.total_failed += 1; + warn!(%error, tx_hash = %raw.tx_hash, "Invalid raw payload; materialization aborted before writes"); + } + } + if result.total_failed != 0 { + return result; + } let dataset_versions = self.resolve_silver_dataset_versions().await; @@ -1478,17 +1489,8 @@ impl Repository { } } - let total = all_token_transfers.len() - + all_native_balance_deltas.len() - + all_decoded_events.len() - + all_hl_fills.len() - + all_hl_funding.len() - + all_hl_positions.len(); - - if total == 0 { - return result; - } - + // Native EVM value/gas flows have no fabricated Silver pre/post balances; + // they still require Gold materialization even when no log was emitted. // Write Silver records to the database. if !all_token_transfers.is_empty() { let n = all_token_transfers.len(); @@ -1590,6 +1592,20 @@ impl Repository { "Bronze-native Silver dataset materialization complete" ); + // Gold foreign keys must use persisted Silver IDs, not fresh parser UUIDs + // that an idempotent Silver upsert may have discarded. + if !all_decoded_events.is_empty() { + let raw_ids: Vec<_> = raw_txs.iter().map(|r| r.id).collect(); + match self.canonical_decoded_events(&raw_ids).await { + Ok(events) => all_decoded_events = events, + Err(error) => { + result.total_failed += all_decoded_events.len(); + warn!(%error, "Could not resolve canonical decoded events"); + return result; + } + } + } + // --- Gold materialization from Silver --- if let Some(wallet) = wallet_address { if !wallet.is_empty() { @@ -1605,6 +1621,12 @@ impl Repository { wallet_target_id, ) .await; + result.total_failed += gold_result.wallet_ledger_failed + + gold_result.balance_history_failed + + gold_result.hl_pnl_summary_failed + + gold_result.hl_trade_history_failed + + gold_result.protocol_events_failed + + gold_result.pool_snapshots_failed; result.gold_wallet_ledger_written = gold_result.wallet_ledger_written; result.gold_balance_history_written = gold_result.balance_history_written; result.gold_hl_pnl_summary_written = gold_result.hl_pnl_summary_written; @@ -1853,11 +1875,14 @@ impl Repository { // --- Token transfers --- for transfer in token_transfers { + if transfer.decimals < 0 { + continue; + } // unknown units remain available in Silver/Raw let from_match = normalize_address_for_comparison(&transfer.from_address) == wallet_norm; let to_match = normalize_address_for_comparison(&transfer.to_address) == wallet_norm; - if !from_match && !to_match { + if from_match == to_match { continue; } @@ -1873,7 +1898,7 @@ impl Repository { if from_match { let key = format!( - "wl:{}:{}:{}:out", + "wl:{wallet_norm}:{}:{}:{}:out", raw_tx_id.map(|u| u.to_string()).unwrap_or_default(), transfer.token_address, transfer.transfer_index @@ -1900,7 +1925,7 @@ impl Repository { if to_match { let key = format!( - "wl:{}:{}:{}:in", + "wl:{wallet_norm}:{}:{}:{}:in", raw_tx_id.map(|u| u.to_string()).unwrap_or_default(), transfer.token_address, transfer.transfer_index @@ -1926,63 +1951,56 @@ impl Repository { } } - // --- Native balance deltas --- + // Separate Solana fees from value movement. HL settlement is represented + // below by fills and funding, not again by its aggregate native deltas. for delta in native_deltas { - if normalize_address_for_comparison(&delta.account_address) != wallet_norm { + if normalize_address_for_comparison(&delta.account_address) != wallet_norm + || delta.network.starts_with("hypercore") + || delta.network.starts_with("hyperliquid") + { continue; } - let raw_tx_id = delta.raw_transaction_id; let raw_tx = raw_tx_id.and_then(|id| raw_tx_map.get(&id)); let tx_hash = raw_tx.map(|r| r.tx_hash.clone()).unwrap_or_default(); let timestamp = raw_tx.map(|r| r.timestamp).unwrap_or(0); - - let key = format!( - "wl:{}:native:{}", - raw_tx_id.map(|u| u.to_string()).unwrap_or_default(), - delta.account_address - ); - - let entry_type = if delta.is_fee_payer { - "fee" + let fee = if delta.is_fee_payer && delta.network.starts_with("solana") { + BigDecimal::from( + raw_tx + .and_then(|r| r.raw_metadata["meta"]["fee"].as_u64()) + .unwrap_or(0), + ) / BigDecimal::from(1_000_000_000u64) } else { - "transfer" + BigDecimal::from(0) }; - - let fee_amount = if delta.is_fee_payer { - // Fee is the absolute value of the delta for fee payers - Some(if delta.delta < BigDecimal::from(0) { - -delta.delta.clone() - } else { - delta.delta.clone() - }) - } else { - None - }; - let fee_asset = if delta.is_fee_payer { - Some(delta.native_token.clone()) - } else { - None - }; - - ledger_records.push(WalletLedgerRecord { - id: Uuid::new_v5(&Uuid::NAMESPACE_URL, key.as_bytes()), - raw_transaction_id: raw_tx_id, - wallet_address: wallet_address.to_string(), - network: delta.network.clone(), - tx_hash, - timestamp, - entry_type: entry_type.to_string(), - asset_symbol: delta.native_token.clone(), - amount: delta.delta.clone(), - counterparty_address: None, - fee_amount, - fee_asset, - cost_basis: None, - proceeds: None, - dataset_version_id: wl_version_id, - created_at: now, - }); + for (kind, amount) in [("transfer", &delta.delta + &fee), ("fee", -fee.clone())] { + if amount == BigDecimal::from(0) { + continue; + } + let key = format!( + "wl:{wallet_norm}:{}:native:{}:{kind}", + raw_tx_id.unwrap_or_default(), + delta.account_address + ); + ledger_records.push(WalletLedgerRecord { + id: Uuid::new_v5(&Uuid::NAMESPACE_URL, key.as_bytes()), + raw_transaction_id: raw_tx_id, + wallet_address: wallet_address.to_string(), + network: delta.network.clone(), + tx_hash: tx_hash.clone(), + timestamp, + entry_type: kind.into(), + asset_symbol: delta.native_token.clone(), + amount, + counterparty_address: None, + fee_amount: (kind == "fee").then(|| fee.clone()), + fee_asset: (kind == "fee").then(|| delta.native_token.clone()), + cost_basis: None, + proceeds: None, + dataset_version_id: wl_version_id, + created_at: now, + }); + } } // --- HL fills --- @@ -1993,7 +2011,7 @@ impl Repository { let trade_id = fill.trade_id.unwrap_or(0); let key = format!( - "wl:{}:fill:{}:{}", + "wl:{wallet_norm}:{}:fill:{}:{}", raw_tx_id.map(|u| u.to_string()).unwrap_or_default(), fill.coin, trade_id @@ -2011,13 +2029,13 @@ impl Repository { wallet_address: wallet_address.to_string(), network: fill.network.clone(), tx_hash, - timestamp: fill.fill_time, + timestamp: raw_tx.map(|r| r.timestamp).unwrap_or(fill.fill_time / 1000), entry_type: "trade".to_string(), asset_symbol: fill.coin.clone(), amount, counterparty_address: None, - fee_amount: fill.fee.clone(), - fee_asset: fill.fee_token.clone(), + fee_amount: None, + fee_asset: None, cost_basis: None, proceeds: None, dataset_version_id: wl_version_id, @@ -2025,6 +2043,51 @@ impl Repository { }); } + // Settlement cash flows are separate from perpetual position quantities. + for fill in hl_fills { + let raw_tx = fill.raw_transaction_id.and_then(|id| raw_tx_map.get(&id)); + for (kind, value, asset) in [ + ( + "fee", + fill.fee.as_ref().map(|v| -v.clone()), + fill.fee_token.as_deref().unwrap_or("USDC"), + ), + ("income", fill.closed_pnl.clone(), "USDC"), + ] { + let Some(amount) = value.filter(|v| *v != BigDecimal::from(0)) else { + continue; + }; + let key = format!( + "wl:{wallet_norm}:{}:fill:{}:{}:{kind}", + fill.raw_transaction_id.unwrap_or_default(), + fill.coin, + fill.trade_id.unwrap_or(0) + ); + ledger_records.push(WalletLedgerRecord { + id: Uuid::new_v5(&Uuid::NAMESPACE_URL, key.as_bytes()), + raw_transaction_id: fill.raw_transaction_id, + wallet_address: wallet_address.into(), + network: fill.network.clone(), + tx_hash: raw_tx.map(|r| r.tx_hash.clone()).unwrap_or_default(), + timestamp: raw_tx.map(|r| r.timestamp).unwrap_or(fill.fill_time / 1000), + entry_type: kind.into(), + asset_symbol: asset.into(), + amount, + counterparty_address: None, + fee_amount: if kind == "fee" { + fill.fee.clone() + } else { + None + }, + fee_asset: (kind == "fee").then(|| asset.into()), + cost_basis: None, + proceeds: None, + dataset_version_id: wl_version_id, + created_at: now, + }); + } + } + // --- HL funding payments --- for funding in hl_funding { let raw_tx_id = funding.raw_transaction_id; @@ -2032,7 +2095,7 @@ impl Repository { let tx_hash = raw_tx.map(|r| r.tx_hash.clone()).unwrap_or_default(); let key = format!( - "wl:{}:funding:{}:{}", + "wl:{wallet_norm}:{}:funding:{}:{}", raw_tx_id.map(|u| u.to_string()).unwrap_or_default(), funding.coin, funding.payment_time @@ -2044,8 +2107,10 @@ impl Repository { wallet_address: wallet_address.to_string(), network: funding.network.clone(), tx_hash, - timestamp: funding.payment_time, - entry_type: format!("funding:{}", funding.coin), + timestamp: raw_tx + .map(|r| r.timestamp) + .unwrap_or(funding.payment_time / 1000), + entry_type: "funding".to_string(), asset_symbol: "USDC".to_string(), amount: funding.amount.clone(), counterparty_address: None, @@ -2058,10 +2123,59 @@ impl Repository { }); } - // Write wallet_ledger records - if !ledger_records.is_empty() { + let mut evm_balance_flows = Vec::new(); + for raw in raw_txs + .iter() + .filter(|r| r.raw_metadata["from"].is_string()) + { + match crate::evm_parser::native_flows(&raw.raw_metadata, wallet_address) { + Ok(flows) => { + let asset = if raw.network.starts_with("hyperevm") { + "HYPE" + } else { + "ETH" + }; + let net: BigDecimal = flows.iter().map(|(_, amount)| amount.clone()).sum(); + if net != BigDecimal::from(0) { + evm_balance_flows.push((raw, asset, net)); + } + for (kind, amount) in flows { + let key = format!("wl:{wallet_norm}:{}:evm:{kind}", raw.id); + ledger_records.push(WalletLedgerRecord { + id: Uuid::new_v5(&Uuid::NAMESPACE_URL, key.as_bytes()), + raw_transaction_id: Some(raw.id), + wallet_address: wallet_address.into(), + network: raw.network.clone(), + tx_hash: raw.tx_hash.clone(), + timestamp: raw.timestamp, + entry_type: kind.into(), + asset_symbol: asset.into(), + fee_amount: (kind == "fee").then(|| -amount.clone()), + fee_asset: (kind == "fee").then(|| asset.into()), + amount, + counterparty_address: None, + cost_basis: None, + proceeds: None, + dataset_version_id: wl_version_id, + created_at: now, + }); + } + } + Err(error) => { + result.wallet_ledger_failed += 1; + warn!(%error, tx_hash = %raw.tx_hash, "Malformed EVM value/gas data"); + } + } + } + + // Replace this wallet's input slice atomically, also repairing prior parser output. + if !raw_txs.is_empty() { let n = ledger_records.len(); - match self.save_wallet_ledger_records(&ledger_records).await { + let inputs: Vec<_> = raw_txs.iter().map(|raw| raw.id).collect(); + match self + .replace_wallet_ledger_records(wallet_address, &inputs, &ledger_records) + .await + { Ok(()) => { result.wallet_ledger_written = n; for record in &ledger_records { @@ -2117,6 +2231,9 @@ impl Repository { let mut balance_events: Vec = Vec::new(); for transfer in token_transfers { + if transfer.decimals < 0 { + continue; + } let from_match = normalize_address_for_comparison(&transfer.from_address) == wallet_norm; let to_match = @@ -2140,7 +2257,19 @@ impl Repository { } else { transfer.amount.clone() }; - let snapshot_id = balance_snapshot_id(&asset, timestamp, &tx_hash, transfer.id); + let source_id = Uuid::new_v5( + &Uuid::NAMESPACE_URL, + format!( + "transfer:{}:{}:{}:{}:{}", + raw_tx_id.unwrap_or_default(), + transfer.token_address, + transfer.from_address, + transfer.to_address, + transfer.transfer_index + ) + .as_bytes(), + ); + let snapshot_id = balance_snapshot_id(&asset, timestamp, &tx_hash, source_id); balance_events.push(BalanceEvent { network: transfer.network.clone(), asset, @@ -2159,8 +2288,18 @@ impl Repository { let raw_tx = raw_tx_id.and_then(|id| raw_tx_map.get(&id)); let tx_hash = raw_tx.map(|r| r.tx_hash.clone()).unwrap_or_default(); let timestamp = raw_tx.map(|r| r.timestamp).unwrap_or(0); + let source_id = Uuid::new_v5( + &Uuid::NAMESPACE_URL, + format!( + "native:{}:{}:{}", + raw_tx_id.unwrap_or_default(), + delta.account_address, + delta.native_token + ) + .as_bytes(), + ); let snapshot_id = - balance_snapshot_id(&delta.native_token, timestamp, &tx_hash, delta.id); + balance_snapshot_id(&delta.native_token, timestamp, &tx_hash, source_id); balance_events.push(BalanceEvent { network: delta.network.clone(), asset: delta.native_token.clone(), @@ -2171,6 +2310,17 @@ impl Repository { }); } + for (raw, asset, delta) in evm_balance_flows { + balance_events.push(BalanceEvent { + network: raw.network.clone(), + asset: asset.into(), + timestamp: raw.timestamp, + tx_hash: raw.tx_hash.clone(), + delta, + snapshot_id: balance_snapshot_id(asset, raw.timestamp, &raw.tx_hash, raw.id), + }); + } + if !balance_events.is_empty() { #[derive(Clone)] struct BalanceRecomputeEvent { @@ -2211,6 +2361,7 @@ impl Repository { } let mut snapshots: Vec = Vec::new(); + let mut snapshot_deltas = HashMap::new(); for ((network, asset), incoming_events) in incoming_by_key { let Some((first_ts, first_id)) = first_event_by_key .get(&(network.clone(), asset.clone())) @@ -2295,8 +2446,29 @@ impl Repository { } }; - let has_existing_suffix = - suffix.iter().any(|snap| !incoming_ids.contains(&snap.id)); + let suffix_ids: Vec<_> = suffix.iter().map(|snap| snap.id).collect(); + let suffix_deltas = match self.balance_snapshot_deltas(&suffix_ids).await { + Ok(deltas) => deltas, + Err(e) => { + let count = incoming_events.len(); + result.balance_history_failed += count; + increment_gold_dataset_network_failures( + &mut result.per_dataset_network_failed, + "balance_history", + &network, + count, + ); + *result + .per_dataset_network_written + .entry(("balance_history".into(), network.clone())) + .or_default() += count; + warn!(error = %e, "Failed to read stored balance deltas"); + continue; + } + }; + let has_existing_suffix = suffix.iter().any(|snap| { + !incoming_ids.contains(&snap.id) && !suffix_deltas.contains_key(&snap.id) + }); if has_existing_suffix { let skipped = incoming_events.len(); result.balance_history_failed += skipped; @@ -2335,10 +2507,20 @@ impl Repository { }) .collect(); - // Only incoming deterministic rows are safe to replay here. Any - // existing suffix rows were fail-closed above because the current - // schema does not store per-row deltas/source order for lossless - // downstream recomputation. + for snapshot in suffix + .into_iter() + .filter(|snap| !incoming_ids.contains(&snap.id)) + { + recompute_events.push(BalanceRecomputeEvent { + id: snapshot.id, + network: snapshot.network, + asset: snapshot.asset_symbol, + timestamp: snapshot.timestamp, + tx_hash: snapshot.tx_hash, + delta: suffix_deltas[&snapshot.id].clone(), + created_at: snapshot.created_at, + }); + } recompute_events.sort_by(|a, b| { a.timestamp.cmp(&b.timestamp).then_with(|| a.id.cmp(&b.id)) @@ -2349,6 +2531,7 @@ impl Repository { .unwrap_or_else(|| BigDecimal::from(0)); for event in recompute_events { balance += &event.delta; + snapshot_deltas.insert(event.id, event.delta.clone()); snapshots.push(BalanceSnapshot { id: event.id, wallet_address: wallet_address.to_string(), @@ -2365,7 +2548,10 @@ impl Repository { if !snapshots.is_empty() { let n = snapshots.len(); - match self.save_balance_snapshots(&snapshots).await { + match self + .save_balance_snapshots_with_deltas(&snapshots, &snapshot_deltas) + .await + { Ok(()) => { result.balance_history_written += n; for snapshot in &snapshots { @@ -2399,6 +2585,28 @@ impl Repository { } } + // Rebuild analytics from all indexed Silver inputs, never a single batch. + // Read persisted IDs as well, so replay cannot fork trade identities. + let hl_network = hl_fills + .first() + .map(|f| f.network.as_str()) + .or_else(|| hl_funding.first().map(|f| f.network.as_str())); + let (indexed_fills, indexed_funding) = if let Some(network) = hl_network { + match self.hl_wallet_inputs(wallet_address, network).await { + Ok(inputs) => inputs, + Err(error) => { + result.hl_pnl_summary_failed += 1; + result.hl_trade_history_failed += 1; + warn!(%error, "Cannot load complete indexed HL analytics inputs"); + return result; + } + } + } else { + (Vec::new(), Vec::new()) + }; + let hl_fills = indexed_fills.as_slice(); + let hl_funding = indexed_funding.as_slice(); + // --- HL PnL summary --- if !hl_fills.is_empty() || !hl_funding.is_empty() { let network = hl_fills @@ -2406,8 +2614,13 @@ impl Repository { .map(|f| f.network.as_str()) .or_else(|| hl_funding.first().map(|f| f.network.as_str())) .unwrap_or("hyperliquid"); - let period_start = raw_txs.iter().map(|r| r.timestamp).min().unwrap_or(0); - let period_end = raw_txs.iter().map(|r| r.timestamp).max().unwrap_or(0); + let times: Vec<_> = hl_fills + .iter() + .map(|f| f.fill_time / 1000) + .chain(hl_funding.iter().map(|f| f.payment_time / 1000)) + .collect(); + let period_start = times.iter().copied().min().unwrap_or(0); + let period_end = times.iter().copied().max().unwrap_or(0); let mut summaries = crate::hl_analytics::compute_pnl_summary( wallet_address, network, @@ -2421,7 +2634,10 @@ impl Repository { } if !summaries.is_empty() { let n = summaries.len(); - match self.save_hl_pnl_summary(&summaries).await { + match self + .replace_hl_pnl_summary(wallet_address, network, &summaries) + .await + { Ok(()) => { result.hl_pnl_summary_written = n; for summary in &summaries { @@ -2464,10 +2680,15 @@ impl Repository { crate::hl_analytics::build_trade_history(wallet_address, network, hl_fills); for t in &mut trades { t.dataset_version_id = hl_trade_version_id; + t.opened_at /= 1000; + t.closed_at /= 1000; } - if !trades.is_empty() { + { let n = trades.len(); - match self.save_hl_trade_history(&trades).await { + match self + .replace_hl_trade_history(wallet_address, network, &trades) + .await + { Ok(()) => { result.hl_trade_history_written = n; for trade in &trades { @@ -2504,8 +2725,20 @@ impl Repository { if !decoded_events.is_empty() { let mut events = crate::protocol_analytics::compute_protocol_events(decoded_events, None); + let event_times: HashMap<_, _> = decoded_events + .iter() + .filter_map(|event| { + event + .raw_transaction_id + .and_then(|id| raw_tx_map.get(&id)) + .map(|raw| (event.id, raw.timestamp)) + }) + .collect(); for e in &mut events { e.dataset_version_id = protocol_events_version_id; + if let Some(time) = e.raw_event_id.and_then(|id| event_times.get(&id)) { + e.timestamp = *time; + } } if !events.is_empty() { let n = events.len(); @@ -2739,7 +2972,7 @@ impl Repository { last_ingestion_run_id: run_id, records_count: total_records_count, gap_ranges: None, - notes: None, + notes: Some("Processing status covers indexed inputs only, not complete chain history. Unknown-decimal ERC20 amounts are excluded from Gold totals.".into()), created_at: now, updated_at: now, }; diff --git a/adapters/src/evm.rs b/adapters/src/evm.rs index 3664800..c40d3f5 100644 --- a/adapters/src/evm.rs +++ b/adapters/src/evm.rs @@ -18,8 +18,8 @@ use spectraplex_core::connector::{ use spectraplex_core::models::{Chain, ChainIngestor, IndexerCheckpoint, Transaction}; use spectraplex_core::provider::{NetworkContext, ProviderCapability}; use spectraplex_core::v2::{ - ChainFamily, EvmTraceType, IndexTarget, IngestionBatch, RawEvmTrace, RawTransaction, - TargetKind, TargetMode, + ChainFamily, Checkpoint, EvmTraceType, IndexTarget, IngestionBatch, RawEvmTrace, + RawTransaction, TargetKind, TargetMode, }; use tracing::{debug, warn}; use uuid::Uuid; @@ -68,8 +68,8 @@ fn next_block_after_i64(last_block: i64) -> Option { .map(|block| block.saturating_add(1)) } -fn default_from_block(latest_block: u64, limit: usize, block_chunk: u64) -> u64 { - latest_block.saturating_sub(usize_to_u64_or_max(limit).saturating_mul(block_chunk)) +fn default_from_block(latest_block: u64, limit: usize, _block_chunk: u64) -> u64 { + latest_block.saturating_sub(usize_to_u64_or_max(limit).saturating_sub(1)) } /// EVM chain adapter that fetches logs and transactions via JSON-RPC. @@ -135,7 +135,7 @@ impl EvmAdapter { /// Set a custom block chunk size for `eth_getLogs` range queries. pub fn with_block_chunk(mut self, chunk: u64) -> Self { - self.block_chunk = chunk; + self.block_chunk = chunk.max(1); self } @@ -192,6 +192,56 @@ impl EvmAdapter { Ok(adapter) } + // A bounded confirmed window, not a claim of irreversible finality. + async fn confirmed_head(&self) -> anyhow::Result { + let expected = match self.network.as_str() { + "ethereum-mainnet" => 1, + "ethereum-sepolia" => 11155111, + "base-mainnet" => 8453, + "base-sepolia" => 84532, + "arbitrum-mainnet" => 42161, + "arbitrum-sepolia" => 421614, + "hyperevm-mainnet" => 999, + "hyperevm-testnet" => 998, + other => anyhow::bail!("No verified EVM chain ID for network {other}"), + }; + self.rate_limiter.until_ready().await; + let actual = self.provider.get_chain_id().await?; + anyhow::ensure!( + actual == expected, + "RPC chain ID mismatch: expected {expected} for {}, got {actual}", + self.network + ); + self.rate_limiter.until_ready().await; + Ok(self.provider.get_block_number().await?.saturating_sub(12)) + } + + async fn scan_window( + &self, + cursor: Option<&serde_json::Value>, + limit: usize, + ) -> anyhow::Result<(u64, u64)> { + anyhow::ensure!( + (1..=100_000).contains(&limit), + "EVM block limit must be 1–100000" + ); + let head = self.confirmed_head().await?; + let from = cursor_to_from_block(cursor, head, limit, self.block_chunk); + let end = head.min(from.saturating_add(limit as u64 - 1)); + Ok((from, end)) + } + + fn scan_checkpoint(&self, target: &IndexTarget, end: u64) -> Checkpoint { + Checkpoint { + id: Uuid::new_v4(), + target_id: target.id, + network: self.network.clone(), + source: "rpc".into(), + cursor: json!({"last_block": end, "confirmations": 12}), + updated_at: Utc::now(), + } + } + // ----------------------------------------------------------------------- // Low-level RPC retry helper // ----------------------------------------------------------------------- @@ -395,28 +445,14 @@ impl EvmAdapter { self.rate_limiter.until_ready().await; // Fetch full block with transactions - let block_opt = self + let block = self .provider .get_block_by_number(block_num.into()) .full() - .await; - - let block = match block_opt { - Ok(Some(b)) => b, - Ok(None) => { - block_num += 1; - continue; - } - Err(e) => { - warn!( - block_number = block_num, - error = %e, - "Failed to fetch block for native tx scan, skipping" - ); - block_num += 1; - continue; - } - }; + .await? + .ok_or_else(|| { + anyhow::anyhow!("Block {block_num} unavailable; checkpoint not advanced") + })?; let block_timestamp = block.header().timestamp(); let block_hash = block.header().hash(); @@ -433,18 +469,19 @@ impl EvmAdapter { let tx_hash = format!("{:#x}", tx.tx_hash()); let value = TransactionTrait::value(tx); - // Fetch receipt for gas data (only for outbound txs) - let mut gas_used: Option = None; - let mut effective_gas_price: Option = None; - if from_match { - self.rate_limiter.until_ready().await; - if let Ok(Some(receipt)) = - self.provider.get_transaction_receipt(tx.tx_hash()).await - { - gas_used = Some(u64_to_u128(receipt.gas_used)); - effective_gas_price = Some(receipt.effective_gas_price); - } - } + self.rate_limiter.until_ready().await; + let receipt = self + .provider + .get_transaction_receipt(tx.tx_hash()) + .await? + .ok_or_else(|| { + anyhow::anyhow!( + "Receipt {tx_hash} unavailable; checkpoint not advanced" + ) + })?; + let receipt_json = serde_json::to_value(&receipt)?; + let gas_used = Some(u64_to_u128(receipt.gas_used)); + let effective_gas_price = Some(receipt.effective_gas_price); records.push(NativeTxRecord { tx_hash, @@ -456,6 +493,7 @@ impl EvmAdapter { block_timestamp, gas_used, effective_gas_price, + receipt: receipt_json, }); } } @@ -528,9 +566,10 @@ impl EvmAdapter { async fn enrich_tx_data( &self, logs: &[Log], - ) -> HashMap { - let mut seen_tx_hashes: HashMap = - HashMap::new(); + ) -> anyhow::Result> { + let mut seen_tx_hashes = HashMap::new(); + let mut block_times = HashMap::new(); + let mut decimal_cache = HashMap::new(); for log in logs { let tx_hash_b256 = match log.transaction_hash { @@ -543,45 +582,98 @@ impl EvmAdapter { continue; } - let mut tx_fields = json!({}); - let mut receipt_fields = json!({}); - + anyhow::ensure!( + !log.removed, + "Provider returned a removed log; retry after reorg" + ); + let block_number = log + .block_number + .ok_or_else(|| anyhow::anyhow!("Log missing block number"))?; + let timestamp = if let Some(time) = block_times.get(&block_number) { + *time + } else { + self.rate_limiter.until_ready().await; + let block = self + .provider + .get_block_by_number(block_number.into()) + .await? + .ok_or_else(|| anyhow::anyhow!("Block {block_number} unavailable"))?; + anyhow::ensure!( + Some(block.header().hash()) == log.block_hash, + "Block changed during ingestion; retry" + ); + let time = block.header().timestamp(); + block_times.insert(block_number, time); + time + }; self.rate_limiter.until_ready().await; - match self.provider.get_transaction_by_hash(tx_hash_b256).await { - Ok(Some(full_tx)) => { - let value = full_tx.inner.value(); - let from = full_tx.inner.signer(); - let to = full_tx.inner.to().map(|a| format!("{a:#x}")); - tx_fields = json!({ - "value": format!("{value:#x}"), - "from": format!("{from:#x}"), - "to": to, - }); - } - Ok(None) => {} - Err(e) => { - warn!(tx_hash = %tx_hash, error = %e, "Failed to fetch transaction"); - } - } - + let full_tx = self + .provider + .get_transaction_by_hash(tx_hash_b256) + .await? + .ok_or_else(|| anyhow::anyhow!("Transaction {tx_hash} unavailable"))?; + let value = full_tx.inner.value(); + let from = full_tx.inner.signer(); + let to = full_tx.inner.to().map(|a| format!("{a:#x}")); + let tx_fields = json!({"value": format!("{value:#x}"), "from": format!("{from:#x}"), "to": to, "timestamp": timestamp}); self.rate_limiter.until_ready().await; - match self.provider.get_transaction_receipt(tx_hash_b256).await { - Ok(Some(receipt)) => { - receipt_fields = json!({ - "gas_used": format!("{:#x}", receipt.gas_used), - "effective_gas_price": format!("{:#x}", receipt.effective_gas_price), - }); + let receipt = self + .provider + .get_transaction_receipt(tx_hash_b256) + .await? + .ok_or_else(|| anyhow::anyhow!("Receipt {tx_hash} unavailable"))?; + let receipt_json = serde_json::to_value(&receipt)?; + let mut decimal_hints = serde_json::Map::new(); + for entry in receipt_json["logs"].as_array().into_iter().flatten() { + if entry["topics"].as_array().map(Vec::len) != Some(3) + || entry["topics"][0] != ERC20_TRANSFER_TOPIC + { + continue; } - Ok(None) => {} - Err(e) => { - warn!(tx_hash = %tx_hash, error = %e, "Failed to fetch receipt"); + let Some(address) = entry["address"].as_str() else { + continue; + }; + let decimals = if let Some(known) = crate::evm_parser::known_token_decimals(address) + { + Some(known) + } else if let Some(cached) = decimal_cache.get(address) { + *cached + } else { + self.rate_limiter.until_ready().await; + let request = json!({"jsonrpc":"2.0", "id":1, "method":"eth_call", "params":[{"to":address,"data":"0x313ce567"}, format!("{block_number:#x}")]}); + let response = match self + .trace_http_client + .post(self.rpc_url.clone()) + .json(&request) + .send() + .await + { + Ok(response) => response.json::().await.ok(), + Err(_) => None, + }; + let value = response + .as_ref() + .and_then(|r| r["result"].as_str()) + .and_then(|s| u32::from_str_radix(s.trim_start_matches("0x"), 16).ok()) + .filter(|d| *d <= 255); + decimal_cache.insert(address.to_string(), value); + value + }; + if let Some(decimals) = decimals { + decimal_hints.insert(address.to_lowercase(), json!(decimals)); } } + let receipt_fields = json!({ + "token_decimals": decimal_hints, + "gas_used": format!("{:#x}", receipt.gas_used), + "effective_gas_price": format!("{:#x}", receipt.effective_gas_price), + "logs": receipt_json["logs"], "status": receipt_json["status"], + }); seen_tx_hashes.insert(tx_hash, (tx_fields, receipt_fields)); } - seen_tx_hashes + Ok(seen_tx_hashes) } // ----------------------------------------------------------------------- @@ -590,9 +682,8 @@ impl EvmAdapter { /// Convert enriched logs into V2 `RawTransaction` records. /// - /// Attaches tx-level fields (value, from, to, gas) only to the first log - /// per transaction hash to avoid duplicate gas fee / value entries in the - /// downstream parser. + /// One canonical row per transaction, containing every receipt log. + /// Otherwise the Bronze unique key silently discards all but one transfer. fn logs_to_raw_transactions( &self, logs: &[Log], @@ -607,8 +698,14 @@ impl EvmAdapter { Some(h) => format!("{h:#x}"), None => continue, }; - - let block_timestamp = log.block_timestamp.unwrap_or(0); + if !emitted_tx_hashes.insert(tx_hash.clone()) { + continue; + } + let block_timestamp = enrichment + .get(&tx_hash) + .and_then(|(fields, _)| fields["timestamp"].as_u64()) + .or(log.block_timestamp) + .unwrap_or(0); let mut raw_metadata = json!({ "log_index": log.log_index, @@ -620,8 +717,7 @@ impl EvmAdapter { "data": format!("0x{}", alloy::hex::encode(log.data().data.as_ref())), }); - let is_first_log = emitted_tx_hashes.insert(tx_hash.clone()); - if is_first_log { + { if let Some((tx_fields, receipt_fields)) = enrichment.get(&tx_hash) { if let Some(obj) = raw_metadata.as_object_mut() { if let Some(tx_obj) = tx_fields.as_object() { @@ -677,17 +773,13 @@ impl EvmAdapter { .as_deref() .ok_or_else(|| anyhow::anyhow!("wallet target must have an address"))?; let wallet_addr: Address = wallet_str.parse()?; - - self.rate_limiter.until_ready().await; - let latest_block = self.provider.get_block_number().await?; - - let from_block = cursor_to_from_block(cursor, latest_block, limit, self.block_chunk); + let (from_block, latest_block) = self.scan_window(cursor, limit).await?; let (logs, native_txs) = self .fetch_wallet_combined(wallet_addr, from_block, latest_block) .await?; - let enrichment = self.enrich_tx_data(&logs).await; + let enrichment = self.enrich_tx_data(&logs).await?; let mut records = self.logs_to_raw_transactions(&logs, &enrichment, "evm-rpc-wallet-backfill"); @@ -715,7 +807,8 @@ impl EvmAdapter { Ok(IngestionBatch { records, - checkpoint: None, + checkpoint: (from_block <= latest_block) + .then(|| self.scan_checkpoint(target, latest_block)), run_metadata: None, }) } @@ -736,23 +829,20 @@ impl EvmAdapter { .as_deref() .ok_or_else(|| anyhow::anyhow!("contract target must have an address"))?; let contract_addr: Address = contract_str.parse()?; - - self.rate_limiter.until_ready().await; - let latest_block = self.provider.get_block_number().await?; - - let from_block = cursor_to_from_block(cursor, latest_block, limit, self.block_chunk); + let (from_block, latest_block) = self.scan_window(cursor, limit).await?; let logs = self .fetch_logs(contract_addr, from_block, latest_block) .await?; - let enrichment = self.enrich_tx_data(&logs).await; + let enrichment = self.enrich_tx_data(&logs).await?; let records = self.logs_to_raw_transactions(&logs, &enrichment, "evm-rpc-contract-backfill"); Ok(IngestionBatch { records, - checkpoint: None, + checkpoint: (from_block <= latest_block) + .then(|| self.scan_checkpoint(target, latest_block)), run_metadata: None, }) } @@ -764,15 +854,12 @@ impl EvmAdapter { /// via the spec's `address_filter` field. async fn topic_filter_backfill( &self, - _target: &IndexTarget, + target: &IndexTarget, spec: &TopicFilterSpec, cursor: Option<&serde_json::Value>, limit: usize, ) -> anyhow::Result { - self.rate_limiter.until_ready().await; - let latest_block = self.provider.get_block_number().await?; - - let from_block = cursor_to_from_block(cursor, latest_block, limit, self.block_chunk); + let (from_block, latest_block) = self.scan_window(cursor, limit).await?; // Parse topic positions from the spec let topic0 = parse_topic_value(spec.topics.first())?; @@ -799,13 +886,14 @@ impl EvmAdapter { ) .await?; - let enrichment = self.enrich_tx_data(&logs).await; + let enrichment = self.enrich_tx_data(&logs).await?; let records = self.logs_to_raw_transactions(&logs, &enrichment, "evm-rpc-topic-filter-backfill"); Ok(IngestionBatch { records, - checkpoint: None, + checkpoint: (from_block <= latest_block) + .then(|| self.scan_checkpoint(target, latest_block)), run_metadata: None, }) } @@ -1065,6 +1153,15 @@ impl Connector for EvmAdapter { ); } + // The target's start block seeds only its first scan. Durable progress + // always wins on subsequent fetches, including empty windows. + let initial_cursor = target + .filter_spec + .as_ref() + .and_then(|s| s.get("from_block")) + .and_then(serde_json::Value::as_u64) + .map(|block| serde_json::json!({"from_block": block})); + let cursor = cursor.or(initial_cursor.as_ref()); match target.kind { TargetKind::Wallet => self.wallet_backfill(target, cursor, limit).await, TargetKind::Contract => self.contract_backfill(target, cursor, limit).await, @@ -1117,7 +1214,7 @@ impl ChainIngestor for EvmAdapter { .fetch_wallet_combined(address, from_block, latest_block) .await?; - let seen_tx_hashes = self.enrich_tx_data(&logs).await; + let seen_tx_hashes = self.enrich_tx_data(&logs).await?; let mut transactions = Vec::new(); let mut emitted_tx_hashes = HashSet::new(); @@ -1213,6 +1310,7 @@ struct NativeTxRecord { block_timestamp: u64, gas_used: Option, effective_gas_price: Option, + receipt: serde_json::Value, } impl NativeTxRecord { @@ -1233,6 +1331,8 @@ impl NativeTxRecord { "to": &self.to, "value": format!("{:#x}", self.value), "source_type": "native_transfer", + "status": self.receipt["status"], + "logs": self.receipt["logs"].as_array().cloned().unwrap_or_default(), }); if let (Some(gas), Some(price)) = (self.gas_used, self.effective_gas_price) { @@ -1294,6 +1394,9 @@ fn cursor_to_from_block( if let Some(last_block) = c.get("last_block").and_then(|v| v.as_u64()) { return last_block.saturating_add(1); } + if let Some(from_block) = c.get("from_block").and_then(serde_json::Value::as_u64) { + return from_block; + } // V1 compat cursor if let Some(last_block) = c.get("last_block").and_then(|v| v.as_i64()) { if let Some(next_block) = next_block_after_i64(last_block) { @@ -1313,6 +1416,33 @@ mod tests { use super::*; use chrono::Utc; + #[test] + fn requested_start_block_is_used_until_a_checkpoint_exists() { + assert_eq!( + cursor_to_from_block( + Some(&serde_json::json!({"from_block": 500})), + 1000, + 50, + 2000 + ), + 500 + ); + assert_eq!( + cursor_to_from_block(Some(&serde_json::json!({"from_block": 0})), 1000, 50, 2000), + 0 + ); + assert_eq!( + cursor_to_from_block( + Some(&serde_json::json!({"from_block": 500, "last_block": 549})), + 1000, + 50, + 2000 + ), + 550 + ); + assert_eq!(cursor_to_from_block(None, 1000, 50, 2000), 951); + } + #[test] fn test_adapter_defaults() { assert_eq!(DEFAULT_BLOCK_CHUNK, 2000); @@ -1419,8 +1549,8 @@ mod tests { #[test] fn test_cursor_to_from_block_no_cursor() { let from = cursor_to_from_block(None, 20_000_000, 10, 2000); - // 20_000_000 - 10 * 2000 = 19_980_000 - assert_eq!(from, 19_980_000); + // Exactly ten blocks, inclusive of the confirmed head. + assert_eq!(from, 19_999_991); } #[test] @@ -1441,7 +1571,7 @@ mod tests { fn test_cursor_to_from_block_v1_negative_uses_default() { let cursor = json!({"v1_compat": true, "last_block": -1i64}); let from = cursor_to_from_block(Some(&cursor), 20_000_000, 10, 2000); - assert_eq!(from, 19_980_000); + assert_eq!(from, 19_999_991); } // -- Connector capabilities -- @@ -1724,6 +1854,7 @@ mod tests { block_timestamp: 1700000000, gas_used: Some(21000), effective_gas_price: Some(1_000_000_000), // 1 gwei + receipt: json!({"status":"0x1", "logs": []}), }; let metadata = record.to_raw_metadata(); @@ -1755,6 +1886,7 @@ mod tests { block_timestamp: 1700000000, gas_used: None, effective_gas_price: None, + receipt: json!({"status":"0x1", "logs": []}), }; let metadata = record.to_raw_metadata(); diff --git a/adapters/src/evm_parser.rs b/adapters/src/evm_parser.rs index 6ccd767..8baf03a 100644 --- a/adapters/src/evm_parser.rs +++ b/adapters/src/evm_parser.rs @@ -196,6 +196,7 @@ fn erc20_approval_topic_addresses(topics: &[serde_json::Value]) -> Option<(Strin /// Handles full uint256 range without truncation. fn hex_to_bigdecimal(hex: &str) -> anyhow::Result { let stripped = hex.strip_prefix("0x").unwrap_or(hex); + anyhow::ensure!(stripped.len() <= 64, "Hex amount exceeds uint256"); if stripped.is_empty() { return Ok(BigDecimal::from(0)); } @@ -211,6 +212,73 @@ fn hex_to_bigdecimal(hex: &str) -> anyhow::Result { Ok(result) } +/// Known top-level native value and execution gas flows only. Never fabricates +/// pre/post account balances or treats internal calls as fully indexed. +pub(crate) fn native_flows( + raw: &serde_json::Value, + wallet: &str, +) -> anyhow::Result> { + let from = raw["from"] + .as_str() + .is_some_and(|v| v.eq_ignore_ascii_case(wallet)); + let to = raw["to"] + .as_str() + .is_some_and(|v| v.eq_ignore_ascii_case(wallet)); + let mut flows = Vec::new(); + if from != to && raw["status"].as_str() != Some("0x0") { + if let Some(value) = raw["value"].as_str() { + let amount = wei_to_eth(hex_to_bigdecimal(value)?); + if amount != BigDecimal::from(0) { + flows.push(("transfer", if from { -amount } else { amount })); + } + } + } + if from { + if let (Some(gas), Some(price)) = ( + raw["gas_used"].as_str(), + raw["effective_gas_price"].as_str(), + ) { + let fee = wei_to_eth(hex_to_bigdecimal(gas)? * hex_to_bigdecimal(price)?); + if fee != BigDecimal::from(0) { + flows.push(("fee", -fee)); + } + } + } + Ok(flows) +} + +#[cfg(test)] +mod native_flow_tests { + use super::*; + use std::str::FromStr; + + #[test] + fn outbound_inbound_failed_and_self_transfers_charge_only_sender_gas() { + let mut raw = serde_json::json!({"from":"0xa", "to":"0xb", "value":"0xde0b6b3a7640000", "gas_used":"0x5208", "effective_gas_price":"0x3b9aca00", "status":"0x1"}); + let fee = BigDecimal::from_str("-0.000021").unwrap(); + assert_eq!( + native_flows(&raw, "0xA").unwrap(), + vec![("transfer", BigDecimal::from(-1)), ("fee", fee.clone())] + ); + assert_eq!( + native_flows(&raw, "0xb").unwrap(), + vec![("transfer", BigDecimal::from(1))] + ); + raw["status"] = serde_json::json!("0x0"); + assert_eq!( + native_flows(&raw, "0xa").unwrap(), + vec![("fee", fee.clone())] + ); + assert!(native_flows(&raw, "0xb").unwrap().is_empty()); + raw["status"] = serde_json::json!("0x1"); + raw["to"] = serde_json::json!("0xa"); + assert_eq!(native_flows(&raw, "0xa").unwrap(), vec![("fee", fee)]); + raw["to"] = serde_json::json!("0xb"); + raw["value"] = serde_json::json!("not-hex"); + assert!(native_flows(&raw, "0xa").is_err()); + } +} + /// Parse a hex string (with optional 0x prefix) into u128. /// Returns zero for empty, invalid, or overflowing values. fn hex_to_u128_or_zero(hex: &str) -> u128 { @@ -227,6 +295,10 @@ fn wei_to_eth(wei: BigDecimal) -> BigDecimal { wei / divisor } +pub(crate) fn known_token_decimals(address: &str) -> Option { + (token_symbol(address) != address).then(|| token_decimals(address)) +} + /// Lookup the number of decimals for well-known ERC-20 tokens. /// Defaults to 18 for unknown tokens. fn token_decimals(contract_address: &str) -> u32 { @@ -415,6 +487,24 @@ pub fn extract_evm_token_transfers( raw_metadata: &serde_json::Value, ) -> Vec { let mut transfers = Vec::new(); + if raw_metadata["status"].as_str() == Some("0x0") { + return transfers; + } + if let Some(logs) = raw_metadata["logs"].as_array() { + for (index, log) in logs.iter().enumerate() { + let mut enriched_log = log.clone(); + enriched_log["token_decimals"] = raw_metadata["token_decimals"].clone(); + for mut transfer in extract_evm_token_transfers(raw_tx_id, network, &enriched_log) { + transfer.transfer_index = log + .get("logIndex") + .or_else(|| log.get("log_index")) + .map(|v| evm_log_index_or_default(v, usize_to_i32_or_max(index))) + .unwrap_or_else(|| usize_to_i32_or_max(index)); + transfers.push(transfer); + } + } + return transfers; + } let topics = match raw_metadata.get("topics").and_then(|t| t.as_array()) { Some(t) => t, @@ -426,7 +516,7 @@ pub fn extract_evm_token_transfers( None => return vec![], }; - if topic0 != ERC20_TRANSFER_TOPIC || topics.len() < 3 { + if topic0 != ERC20_TRANSFER_TOPIC || topics.len() != 3 { return vec![]; } @@ -453,9 +543,15 @@ pub fn extract_evm_token_transfers( .unwrap_or("unknown") .to_string(); - let decimals = token_decimals(&token_address); + let decimals = raw_metadata["token_decimals"][token_address.to_lowercase()] + .as_u64() + .and_then(|n| u32::try_from(n).ok()) + .filter(|n| *n <= 255) + .or_else(|| known_token_decimals(&token_address)); let symbol = token_symbol(&token_address); - let normalized = normalize_bigdecimal(amount_bd, decimals); + let normalized = decimals + .map(|d| normalize_bigdecimal(amount_bd.clone(), d)) + .unwrap_or(amount_bd); transfers.push(TokenTransfer { id: Uuid::new_v4(), @@ -466,7 +562,7 @@ pub fn extract_evm_token_transfers( from_address: from, to_address: to, amount: normalized, - decimals: token_decimals_i32(decimals), + decimals: decimals.map(token_decimals_i32).unwrap_or(-1), transfer_index: 0, dataset_version_id: None, created_at: Utc::now(), @@ -496,7 +592,11 @@ pub fn extract_evm_decoded_events( let mut events = Vec::new(); // Handle single-log format (topics/data/address at top level) - if let Some(topics) = raw_metadata.get("topics").and_then(|t| t.as_array()) { + if let Some(topics) = raw_metadata + .get("topics") + .and_then(|t| t.as_array()) + .filter(|_| !raw_metadata["logs"].is_array()) + { let address = raw_metadata .get("address") .and_then(|a| a.as_str()) diff --git a/adapters/src/hyperliquid.rs b/adapters/src/hyperliquid.rs index 0a245c6..5aa7770 100644 --- a/adapters/src/hyperliquid.rs +++ b/adapters/src/hyperliquid.rs @@ -242,8 +242,21 @@ impl HyperliquidAdapter { }; let resp = self.client.post(&self.base_url).json(&body).send().await?; let resp = ensure_success_response(resp, "userFunding").await?; - let funding: Vec = resp.json().await?; - Ok(funding) + let values: Vec = resp.json().await?; + values + .into_iter() + .map(|value| { + let normalized = if value["delta"].is_object() { + let mut delta = value["delta"].clone(); + delta["time"] = value["time"].clone(); + delta["hash"] = value["hash"].clone(); + delta + } else { + value + }; + Ok(serde_json::from_value(normalized)?) + }) + .collect() } pub async fn fetch_user_ledger_updates( @@ -307,70 +320,67 @@ impl HyperliquidAdapter { .as_deref() .ok_or_else(|| anyhow::anyhow!("wallet target must have an address"))?; - let resume_ms = cursor_to_start_time_ms(cursor); - let resume_ms_u64 = i64_to_u64_or_zero(resume_ms); - let network = &target.network; + anyhow::ensure!( + (1..=100_000).contains(&limit), + "Hyperliquid limit must be 1–100000" + ); + let mut next = cursor.cloned().unwrap_or_else(|| serde_json::json!({})); let mut records = Vec::new(); - - // 1. Fills — no startTime param, filter client-side - let fills = self.fetch_user_fills(wallet_str).await?; - for fill in fills.iter().filter(|f| f.time >= resume_ms_u64).take(limit) { - let raw = serde_json::to_value(fill)?; - records.push(RawTransaction { - id: Uuid::new_v4(), - network: network.clone(), - tx_hash: fill.hash.clone(), - timestamp: unix_ms_to_secs_i64(fill.time), - block_number: None, - raw_metadata: serde_json::json!({ "type": "fill", "data": raw }), - source: "hl-rest-wallet-backfill".to_string(), - ingestion_run_id: None, - ingested_at: Utc::now(), - }); - } - - // 2. Funding payments - let remaining = limit.saturating_sub(records.len()); - if remaining > 0 { - let funding = self.fetch_user_funding(wallet_str, resume_ms).await?; - for entry in funding.iter().take(remaining) { - let raw = serde_json::to_value(entry)?; - let hash = entry - .hash - .clone() - .unwrap_or_else(|| format!("funding-{}-{}", entry.coin, entry.time)); - records.push(RawTransaction { - id: Uuid::new_v4(), - network: network.clone(), - tx_hash: hash, - timestamp: unix_ms_to_secs_i64(entry.time), - block_number: None, - raw_metadata: serde_json::json!({ "type": "funding", "data": raw }), - source: "hl-rest-wallet-backfill".to_string(), - ingestion_run_id: None, - ingested_at: Utc::now(), - }); - } - } - - // 3. Non-funding ledger updates - let remaining = limit.saturating_sub(records.len()); - if remaining > 0 { - let ledger_updates = self - .fetch_user_ledger_updates(wallet_str, resume_ms) - .await?; - for update in ledger_updates.iter().take(remaining) { - let raw = serde_json::to_value(update)?; + // Independent cursors prevent a full fills page from starving older funding + // and ledger updates. Never truncate in the middle of a millisecond. + for (kind, field) in [ + ("fill", "fills_ms"), + ("funding", "funding_ms"), + ("ledger_update", "ledger_ms"), + ] { + let start = next[field].as_i64().map(resume_after_ms).unwrap_or(0); + let values: Vec = match kind { + "fill" => { + let resp = self.client.post(&self.base_url).json(&serde_json::json!({ + "type": "userFillsByTime", "user": wallet_str, "startTime": start, "aggregateByTime": false, + })).send().await?; + let fills: Vec = ensure_success_response(resp, "userFillsByTime") + .await? + .json() + .await?; + fills + .into_iter() + .map(serde_json::to_value) + .collect::>()? + } + "funding" => self + .fetch_user_funding(wallet_str, start) + .await? + .into_iter() + .map(serde_json::to_value) + .collect::>()?, + _ => self + .fetch_user_ledger_updates(wallet_str, start) + .await? + .into_iter() + .map(serde_json::to_value) + .collect::>()?, + }; + let values = bounded_time_window( + values, + start, + limit, + if kind == "fill" { 2000 } else { 500 }, + )?; + for data in values { + let time = data["time"] + .as_i64() + .ok_or_else(|| anyhow::anyhow!("Invalid Hyperliquid event time"))?; + next[field] = serde_json::json!(time); + // An order hash can have multiple fills and appear in both users' + // histories. Keep the original in data.hash, not as the unique key. + let identity = serde_json::to_vec(&(wallet_str.to_lowercase(), kind, &data))?; + let event_id = Uuid::new_v5(&Uuid::NAMESPACE_URL, &identity); records.push(RawTransaction { - id: Uuid::new_v4(), - network: network.clone(), - tx_hash: update.hash.clone(), - timestamp: unix_ms_to_secs_i64(update.time), - block_number: None, - raw_metadata: serde_json::json!({ "type": "ledger_update", "data": raw }), - source: "hl-rest-wallet-backfill".to_string(), - ingestion_run_id: None, - ingested_at: Utc::now(), + id: Uuid::new_v4(), network: target.network.clone(), tx_hash: format!("hl-{kind}-{event_id}"), + timestamp: time / 1000, block_number: None, + raw_metadata: serde_json::json!({"type": kind, "wallet": wallet_str, "data": data}), + source: "hl-rest-wallet-backfill".into(), ingestion_run_id: None, ingested_at: Utc::now(), }); } } @@ -383,7 +393,14 @@ impl HyperliquidAdapter { Ok(IngestionBatch { records, - checkpoint: None, + checkpoint: Some(spectraplex_core::v2::Checkpoint { + id: Uuid::new_v4(), + target_id: target.id, + network: target.network.clone(), + source: "rest".into(), + cursor: next, + updated_at: Utc::now(), + }), run_metadata: None, }) } @@ -442,6 +459,39 @@ impl HyperliquidAdapter { // Cursor helpers // --------------------------------------------------------------------------- +fn bounded_time_window( + mut values: Vec, + start: i64, + limit: usize, + provider_cap: usize, +) -> anyhow::Result> { + anyhow::ensure!( + values + .iter() + .all(|v| v["time"].as_i64().is_some_and(|t| t >= 0)), + "Invalid event timestamp" + ); + let saturated = values.len() >= provider_cap; + values.retain(|v| v["time"].as_i64().unwrap() >= start); + values.sort_by_key(|v| v["time"].as_i64().unwrap()); + if saturated { + // The provider might have more events at its last timestamp. Defer that + // entire timestamp to the next request instead of silently dropping ties. + if let Some(last) = values.last().map(|v| v["time"].as_i64().unwrap()) { + values.retain(|v| v["time"].as_i64().unwrap() < last); + anyhow::ensure!( + !values.is_empty(), + "Provider page saturated at one timestamp; cannot advance safely" + ); + } + } + if values.len() > limit { + let boundary = values[limit - 1]["time"].as_i64().unwrap(); + values.retain(|v| v["time"].as_i64().unwrap() <= boundary); + } + Ok(values) +} + /// Extract start time in milliseconds from a V2 checkpoint cursor. /// /// Cursor shape: `{"last_time_ms": i64}`. @@ -971,6 +1021,22 @@ mod tests { // V2 wallet backfill with mock server // ----------------------------------------------------------------------- + #[test] + fn bounded_windows_keep_ties_and_defer_saturated_tail() { + let values = vec![ + serde_json::json!({"time":3}), + serde_json::json!({"time":2}), + serde_json::json!({"time":1}), + serde_json::json!({"time":2}), + ]; + let page = bounded_time_window(values.clone(), 0, 2, 500).unwrap(); + assert_eq!(page.len(), 3); + assert_eq!(page.last().unwrap()["time"], 2); + let page = bounded_time_window(values, 0, 50, 4).unwrap(); + assert_eq!(page.len(), 3); + assert!(bounded_time_window(vec![serde_json::json!({"time":1}); 500], 0, 50, 500).is_err()); + } + #[tokio::test] async fn test_v2_wallet_backfill_with_mock_server() { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -1033,7 +1099,12 @@ mod tests { // Verify record types assert_eq!(batch.records[0].raw_metadata["type"], "fill"); - assert_eq!(batch.records[0].tx_hash, "0xtest1"); + assert!(batch.records[0].tx_hash.starts_with("hl-fill-")); + assert_eq!(batch.records[0].raw_metadata["data"]["hash"], "0xtest1"); + let checkpoint = batch.checkpoint.as_ref().unwrap(); + for field in ["fills_ms", "funding_ms", "ledger_ms"] { + assert_eq!(checkpoint.cursor[field], 1700000000000_i64); + } assert_eq!(batch.records[1].raw_metadata["type"], "funding"); assert_eq!(batch.records[2].raw_metadata["type"], "ledger_update"); @@ -1089,8 +1160,8 @@ mod tests { ); // Cursor at the same time as the mock fill (1700000000000ms) - // cursor_to_start_time_ms adds 1, so resume_ms = 1700000000001 > fill time - let cursor = serde_json::json!({ "last_time_ms": 1700000000000_i64 }); + // Each stream resumes independently; funding is not skipped by the fill cursor. + let cursor = serde_json::json!({ "fills_ms": 1700000000000_i64 }); let batch = adapter.backfill(&target, Some(&cursor), 100).await.unwrap(); // Fill should be filtered out diff --git a/adapters/src/hyperliquid_parser.rs b/adapters/src/hyperliquid_parser.rs index f71ebd1..861f73e 100644 --- a/adapters/src/hyperliquid_parser.rs +++ b/adapters/src/hyperliquid_parser.rs @@ -82,7 +82,7 @@ fn parse_fill(tx: &Transaction, data: &serde_json::Value) -> anyhow::Result anyhow::Result BigDecimal::from(0) { + EntryType::Income + } else { + EntryType::Fee + }, amount, - entry_type: EntryType::Fee, fiat_value: None, }]) } @@ -366,7 +370,7 @@ pub fn extract_hyperliquid_native_balance_deltas( }; // Total USDC delta from this fill = -fee + closed_pnl - let delta = &pnl - &fee.abs(); + let delta = &pnl - &fee; if delta == BigDecimal::from(0) { return vec![]; diff --git a/adapters/src/ledger_derivation.rs b/adapters/src/ledger_derivation.rs index 3e6c418..a90c468 100644 --- a/adapters/src/ledger_derivation.rs +++ b/adapters/src/ledger_derivation.rs @@ -220,7 +220,7 @@ pub fn derive_ledger_from_hl_fills( user_id, wallet_address: wallet.to_string(), asset_symbol: fee_token.to_string(), - amount: -fee.abs(), + amount: -fee.clone(), entry_type: EntryType::Fee, fiat_value: None, }); @@ -250,8 +250,7 @@ pub fn derive_ledger_from_hl_fills( /// Derive ledger entries from Silver `HlFundingPayment` records. /// -/// Converts each funding payment into a `Fee` entry, consistent with -/// V1 `parse_funding` behavior. +/// Funding received is income; funding paid is an expense. pub fn derive_ledger_from_hl_funding( wallet: &str, user_id: Uuid, @@ -269,7 +268,11 @@ pub fn derive_ledger_from_hl_funding( wallet_address: wallet.to_string(), asset_symbol: "USDC".to_string(), amount: payment.amount.clone(), - entry_type: EntryType::Fee, + entry_type: if payment.amount > BigDecimal::from(0) { + EntryType::Income + } else { + EntryType::Fee + }, fiat_value: None, }); *entry_offset += 1; @@ -673,9 +676,9 @@ pub fn derive_wallet_ledger_from_hl_fills( timestamp, entry_type: "fee".to_string(), asset_symbol: fee_token.to_string(), - amount: -fee.abs(), + amount: -fee.clone(), counterparty_address: None, - fee_amount: Some(fee.abs()), + fee_amount: Some(fee.clone()), fee_asset: Some(fee_token.to_string()), cost_basis: None, proceeds: None, @@ -1381,7 +1384,7 @@ mod tests { assert_eq!(entries.len(), 1); assert_eq!(entries[0].asset_symbol, "USDC"); assert_eq!(entries[0].amount, BigDecimal::from_str("5.0").unwrap()); - assert!(matches!(entries[0].entry_type, EntryType::Fee)); + assert!(matches!(entries[0].entry_type, EntryType::Income)); } #[test] diff --git a/adapters/src/lib.rs b/adapters/src/lib.rs index 66f0c1c..100f18c 100644 --- a/adapters/src/lib.rs +++ b/adapters/src/lib.rs @@ -11,11 +11,149 @@ pub mod protocol_analytics; pub mod repo; pub mod solana; pub mod solana_grpc; +mod solana_history; pub mod solana_parser; pub mod v2_repo; +mod workbench_repo; use uuid::Uuid; +/// Validate supported payload shapes and amounts before import or materialization. +pub fn validate_import_payload( + family: spectraplex_core::v2::ChainFamily, + raw: &serde_json::Value, +) -> anyhow::Result<()> { + use spectraplex_core::v2::ChainFamily; + anyhow::ensure!(raw.is_object(), "raw_metadata must be an object"); + let decimal = |text: &str| -> anyhow::Result<()> { + anyhow::ensure!(text.len() <= 256, "Decimal amount is too long"); + let _: bigdecimal::BigDecimal = text.parse()?; + Ok(()) + }; + match family { + ChainFamily::Solana => { + let tx: solana_transaction_status::EncodedConfirmedTransactionWithStatusMeta = + serde_json::from_value(raw.clone())?; + let meta = tx + .transaction + .meta + .ok_or_else(|| anyhow::anyhow!("Solana metadata is required"))?; + anyhow::ensure!( + meta.pre_balances.len() == meta.post_balances.len(), + "Solana balance array length mismatch" + ); + for balances in [&meta.pre_token_balances, &meta.post_token_balances] { + if let solana_transaction_status::option_serializer::OptionSerializer::Some( + balances, + ) = balances + { + for balance in balances { + let _: u64 = balance.ui_token_amount.amount.parse()?; + } + } + } + } + ChainFamily::Evm => { + anyhow::ensure!( + (raw["topics"].is_array() && raw["data"].is_string() && raw["address"].is_string()) + || (raw["from"].is_string() && raw["value"].is_string()) + || raw["logs"].is_array(), + "EVM payload must be an RPC log or transaction" + ); + fn check_evm_fields(raw: &serde_json::Value) -> anyhow::Result<()> { + anyhow::ensure!(raw.is_object(), "EVM log must be an object"); + for field in ["value", "gas_used", "effective_gas_price", "data"] { + if let Some(value) = raw.get(field) { + let text = value + .as_str() + .ok_or_else(|| anyhow::anyhow!("{field} must be a hex string"))?; + anyhow::ensure!( + text.starts_with("0x") + && text[2..].bytes().all(|b| b.is_ascii_hexdigit()), + "Invalid hex in {field}" + ); + if field != "data" { + anyhow::ensure!( + text.len() <= 66 && text.len() > 2, + "Invalid uint256 in {field}" + ); + } + } + } + if let Some(topics) = raw.get("topics") { + let topics = topics + .as_array() + .ok_or_else(|| anyhow::anyhow!("topics must be an array"))?; + anyhow::ensure!( + topics.len() <= 4 + && topics + .iter() + .all(|t| t.as_str().is_some_and(|s| s.len() == 66 + && s.starts_with("0x") + && s[2..].bytes().all(|b| b.is_ascii_hexdigit()))), + "Invalid EVM topics" + ); + } + Ok(()) + } + check_evm_fields(raw)?; + if let Some(logs) = raw["logs"].as_array() { + for log in logs { + check_evm_fields(log)?; + } + } + if let Some(hints) = raw["token_decimals"].as_object() { + anyhow::ensure!( + hints.values().all(|v| v.as_u64().is_some_and(|n| n <= 255)), + "Token decimals must be integers 0–255" + ); + } + } + ChainFamily::Hyperliquid => match raw["type"].as_str() { + Some("fill") => { + let fill: hyperliquid::HlFill = serde_json::from_value(raw["data"].clone())?; + for text in [ + Some(fill.px.as_str()), + Some(fill.sz.as_str()), + fill.fee.as_deref(), + fill.closed_pnl.as_deref(), + ] + .into_iter() + .flatten() + { + decimal(text)?; + } + } + Some("funding") => { + let funding: hyperliquid::HlFundingEntry = + serde_json::from_value(raw["data"].clone())?; + decimal(&funding.usdc)?; + if let Some(rate) = funding.funding_rate { + decimal(&rate)?; + } + } + Some("ledger_update") => { + let update: hyperliquid::HlLedgerUpdate = + serde_json::from_value(raw["data"].clone())?; + if matches!(update.delta["type"].as_str(), Some("deposit" | "withdraw")) { + decimal( + update.delta["usdc"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("Ledger usdc amount is required"))?, + )?; + } + } + Some("funding_rate") => { + let rate: hyperliquid::HlFundingRate = serde_json::from_value(raw["data"].clone())?; + decimal(&rate.funding_rate)?; + decimal(&rate.premium)?; + } + _ => anyhow::bail!("Unknown Hyperliquid payload type"), + }, + } + Ok(()) +} + const LEDGER_ENTRY_NS: Uuid = Uuid::from_bytes([ 0x6b, 0xa7, 0xb8, 0x10, 0x9d, 0xad, 0x11, 0xd1, 0x80, 0xb4, 0x00, 0xc0, 0x4f, 0xd4, 0x30, 0xc8, ]); diff --git a/adapters/src/protocol_analytics.rs b/adapters/src/protocol_analytics.rs index b6a5be0..ec0cead 100644 --- a/adapters/src/protocol_analytics.rs +++ b/adapters/src/protocol_analytics.rs @@ -94,6 +94,11 @@ pub fn compute_pool_snapshots( token1: (&str, Option<&str>), ) -> Vec { let now = chrono::Utc::now(); + // Never publish a fabricated pool with unidentified assets. The generic + // wallet decoder does not establish token pairs or authoritative reserves. + if token0.0.is_empty() || token1.0.is_empty() { + return Vec::new(); + } // Derive a single snapshot from the latest event let Some(latest) = decoded_events.iter().max_by_key(|e| e.created_at) else { diff --git a/adapters/src/repo.rs b/adapters/src/repo.rs index c580421..8cc3786 100644 --- a/adapters/src/repo.rs +++ b/adapters/src/repo.rs @@ -116,6 +116,11 @@ impl Repository { &self.pool } + pub async fn check_connection(&self) -> anyhow::Result<()> { + sqlx::query("SELECT 1").execute(&self.pool).await?; + Ok(()) + } + /// Batch size for chunked inserts. const BATCH_SIZE: usize = 500; diff --git a/adapters/src/solana.rs b/adapters/src/solana.rs index c2a2f0f..75063c7 100644 --- a/adapters/src/solana.rs +++ b/adapters/src/solana.rs @@ -1,20 +1,23 @@ +#[cfg(test)] use std::str::FromStr; use std::sync::Arc; use std::time::Duration; +#[cfg(test)] use chrono::Utc; -use solana_client::rpc_client::{GetConfirmedSignaturesForAddress2Config, RpcClient}; -use solana_sdk::{pubkey::Pubkey, signature::Signature}; -use solana_transaction_status::UiTransactionEncoding; +use solana_client::nonblocking::rpc_client::RpcClient; +#[cfg(test)] +use solana_sdk::signature::Signature; use spectraplex_core::connector::{Connector, ConnectorCapabilities}; use spectraplex_core::models::{Chain, ChainIngestor, IndexerCheckpoint, Transaction}; use spectraplex_core::provider::{NetworkContext, ProviderCapability}; -use spectraplex_core::v2::{ - ChainFamily, IndexTarget, IngestionBatch, RawTransaction, TargetKind, TargetMode, -}; -use tracing::{info, warn}; +use spectraplex_core::v2::{ChainFamily, IndexTarget, IngestionBatch, TargetKind, TargetMode}; + +#[cfg(test)] +use spectraplex_core::v2::RawTransaction; use uuid::Uuid; +#[cfg(test)] fn u64_to_i64_or_max(value: u64) -> i64 { i64::try_from(value).unwrap_or(i64::MAX) } @@ -76,63 +79,31 @@ impl ChainIngestor for SolanaAdapter { user_id: Uuid, checkpoint: Option<&IndexerCheckpoint>, ) -> anyhow::Result> { - let client = self.client.clone(); - let wallet = wallet.to_string(); - let until_sig = checkpoint - .and_then(|cp| cp.last_signature.clone()) - .and_then(|s| Signature::from_str(&s).ok()); - - tokio::task::spawn_blocking(move || { - let pubkey = Pubkey::from_str(&wallet)?; - let config = GetConfirmedSignaturesForAddress2Config { - until: until_sig, - limit: Some(limit), - ..Default::default() - }; - let signatures = client.get_signatures_for_address_with_config(&pubkey, config)?; - - let mut transactions = Vec::new(); - - for sig_info in signatures.iter().take(limit) { - let sig = Signature::from_str(&sig_info.signature)?; - - match client.get_transaction(&sig, UiTransactionEncoding::Json) { - Ok(tx) => { - let raw_metadata = match serde_json::to_value(&tx) { - Ok(v) => v, - Err(e) => { - warn!( - tx_hash = %sig_info.signature, - error = %e, - "Failed to serialize transaction metadata, using empty object" - ); - serde_json::Value::Object(Default::default()) - } - }; - - transactions.push(Transaction { - id: Uuid::new_v4(), - user_id, - wallet_address: wallet.to_string(), - timestamp: tx.block_time.unwrap_or(0), - tx_hash: sig_info.signature.clone(), - chain: Chain::Solana, - raw_metadata, - }); - } - Err(e) => { - warn!( - tx_hash = %sig_info.signature, - error = %e, - "Failed to fetch transaction, skipping" - ); - } - } - } - - Ok(transactions) - }) - .await? + let cursor = checkpoint.map( + |cp| serde_json::json!({"last_signature": cp.last_signature, "incremental": true}), + ); + let batch = crate::solana_history::fetch( + self.client.clone(), + wallet.into(), + self.network.clone(), + cursor, + limit, + Uuid::nil(), + ) + .await?; + Ok(batch + .records + .into_iter() + .map(|raw| Transaction { + id: raw.id, + user_id, + wallet_address: wallet.into(), + timestamp: raw.timestamp, + tx_hash: raw.tx_hash, + chain: Chain::Solana, + raw_metadata: raw.raw_metadata, + }) + .collect()) } } @@ -141,6 +112,7 @@ impl ChainIngestor for SolanaAdapter { // --------------------------------------------------------------------------- /// Extract a cursor with {last_signature} for Solana RPC pagination. +#[cfg(test)] fn cursor_to_until_sig(cursor: Option<&serde_json::Value>) -> Option { cursor .and_then(|c| c.get("last_signature")) @@ -183,82 +155,15 @@ impl Connector for SolanaAdapter { .as_deref() .ok_or_else(|| anyhow::anyhow!("wallet target must have an address"))?; - let client = self.client.clone(); - let wallet = wallet_str.to_string(); - let network = target.network.clone(); - let until_sig = cursor_to_until_sig(cursor); - - info!( - "Solana RPC V2 backfill: fetching up to {} signatures for wallet {}", - limit, wallet - ); - - let records = tokio::task::spawn_blocking(move || { - let pubkey = Pubkey::from_str(&wallet)?; - let config = GetConfirmedSignaturesForAddress2Config { - until: until_sig, - limit: Some(limit), - ..Default::default() - }; - let signatures = client.get_signatures_for_address_with_config(&pubkey, config)?; - - let mut records = Vec::new(); - - for sig_info in signatures.iter().take(limit) { - let sig = Signature::from_str(&sig_info.signature)?; - - match client.get_transaction(&sig, UiTransactionEncoding::Json) { - Ok(tx) => { - let raw_metadata = match serde_json::to_value(&tx) { - Ok(v) => v, - Err(e) => { - warn!( - tx_hash = %sig_info.signature, - error = %e, - "Failed to serialize transaction metadata, using empty object" - ); - serde_json::Value::Object(Default::default()) - } - }; - - let slot = tx.slot; - - records.push(RawTransaction { - id: Uuid::new_v4(), - network: network.clone(), - tx_hash: sig_info.signature.clone(), - timestamp: tx.block_time.unwrap_or(0), - block_number: Some(u64_to_i64_or_max(slot)), - raw_metadata, - source: "solana-rpc-wallet-backfill".to_string(), - ingestion_run_id: None, - ingested_at: Utc::now(), - }); - } - Err(e) => { - warn!( - tx_hash = %sig_info.signature, - error = %e, - "Failed to fetch transaction, skipping" - ); - } - } - } - - Ok::, anyhow::Error>(records) - }) - .await??; - - info!( - "Solana RPC V2 backfill: collected {} records", - records.len() - ); - - Ok(IngestionBatch { - records, - checkpoint: None, - run_metadata: None, - }) + crate::solana_history::fetch( + self.client.clone(), + wallet_str.into(), + target.network.clone(), + cursor.cloned(), + limit, + target.id, + ) + .await } } diff --git a/adapters/src/solana_history.rs b/adapters/src/solana_history.rs new file mode 100644 index 0000000..87fed36 --- /dev/null +++ b/adapters/src/solana_history.rs @@ -0,0 +1,229 @@ +use anyhow::Context; +use solana_client::{ + nonblocking::rpc_client::RpcClient, rpc_client::GetConfirmedSignaturesForAddress2Config, + rpc_config::RpcTransactionConfig, rpc_response::RpcConfirmedTransactionStatusWithSignature, +}; +use solana_sdk::{pubkey::Pubkey, signature::Signature}; +use solana_transaction_status::UiTransactionEncoding; +use spectraplex_core::v2::{Checkpoint, IngestionBatch, RawTransaction}; +use std::{str::FromStr, sync::Arc}; +use uuid::Uuid; + +async fn signatures( + limit: usize, + mut before: Option, + until: Option, + reject_overflow: bool, + mut fetch: F, +) -> anyhow::Result<(Vec, bool)> +where + F: FnMut(GetConfirmedSignaturesForAddress2Config) -> Fut + Send, + Fut: std::future::Future< + Output = anyhow::Result>, + > + Send, +{ + anyhow::ensure!( + limit > 0 && limit <= 100_000, + "Solana limit must be 1–100000" + ); + let mut result = Vec::new(); + loop { + let page_limit = (limit - result.len()).min(1000); + let page = fetch(GetConfirmedSignaturesForAddress2Config { + before, + until, + limit: Some(page_limit), + ..Default::default() + }) + .await?; + anyhow::ensure!( + page.len() <= page_limit, + "Solana provider exceeded requested page size" + ); + let count = page.len(); + if let Some(last) = page.last() { + let next = Signature::from_str(&last.signature)?; + anyhow::ensure!( + Some(next) != before, + "Solana provider repeated pagination cursor" + ); + before = Some(next); + } + result.extend(page); + if count < page_limit { + return Ok((result, true)); + } + if result.len() == limit { + if reject_overflow { + let extra = fetch(GetConfirmedSignaturesForAddress2Config { + before, + until, + limit: Some(1), + ..Default::default() + }) + .await?; + anyhow::ensure!(extra.is_empty(), "More new Solana activity than ingest_limit; increase ingest_limit and retry (checkpoint was not advanced)"); + } + return Ok((result, false)); + } + } +} + +pub(crate) async fn fetch( + client: Arc, + wallet: String, + network: String, + cursor: Option, + limit: usize, + target_id: Uuid, +) -> anyhow::Result { + let pubkey = Pubkey::from_str(&wallet)?; + let cursor = cursor.unwrap_or_default(); + let incremental = cursor["incremental"].as_bool().unwrap_or(false); + let parse = |name: &str| -> anyhow::Result> { + cursor[name] + .as_str() + .map(Signature::from_str) + .transpose() + .map_err(Into::into) + }; + let head = parse("last_signature")?; + let before = if incremental { + None + } else { + parse("before_signature")? + }; + let until = if incremental { head } else { None }; + let (signatures, exhausted) = signatures( + limit, + before, + until, + incremental && head.is_some(), + |config| { + let client = client.clone(); + async move { + Ok(client + .get_signatures_for_address_with_config(&pubkey, config) + .await?) + } + }, + ) + .await?; + let mut records = Vec::with_capacity(signatures.len()); + for info in &signatures { + let signature = Signature::from_str(&info.signature)?; + let tx = client + .get_transaction_with_config( + &signature, + RpcTransactionConfig { + encoding: Some(UiTransactionEncoding::Json), + max_supported_transaction_version: Some(0), + ..Default::default() + }, + ) + .await + .with_context(|| { + format!( + "Failed to fetch {}; checkpoint not advanced", + info.signature + ) + })?; + let timestamp = tx + .block_time + .ok_or_else(|| anyhow::anyhow!("Missing blockTime for {}", info.signature))?; + records.push(RawTransaction { + id: Uuid::new_v4(), + network: network.clone(), + tx_hash: info.signature.clone(), + timestamp, + block_number: Some(i64::try_from(tx.slot)?), + raw_metadata: serde_json::to_value(tx)?, + source: "solana-rpc-wallet-backfill".into(), + ingestion_run_id: None, + ingested_at: chrono::Utc::now(), + }); + } + let new_head = if incremental || head.is_none() { + signatures + .first() + .map(|s| s.signature.clone()) + .or_else(|| head.map(|h| h.to_string())) + } else { + head.map(|h| h.to_string()) + }; + let new_before = if incremental && cursor["before_signature"].is_string() { + cursor["before_signature"].as_str().map(str::to_owned) + } else { + signatures + .last() + .map(|s| s.signature.clone()) + .or_else(|| before.map(|s| s.to_string())) + }; + let checkpoint = Checkpoint { + id: Uuid::new_v4(), + target_id, + network, + source: "rpc".into(), + updated_at: chrono::Utc::now(), + cursor: serde_json::json!({"last_signature": new_head, "before_signature": new_before, + "last_slot": signatures.first().map(|s| s.slot), "history_exhausted": if incremental { cursor["history_exhausted"].as_bool().unwrap_or(false) } else { exhausted }}), + }; + Ok(IngestionBatch { + records, + checkpoint: Some(checkpoint), + run_metadata: None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + fn info(index: usize) -> RpcConfirmedTransactionStatusWithSignature { + let mut bytes = [0u8; 64]; + bytes[..8].copy_from_slice(&index.to_le_bytes()); + serde_json::from_value(serde_json::json!({"signature": bs58::encode(bytes).into_string(), "slot": index, "err": null, "memo": null, "blockTime": 1700000000, "confirmationStatus": "finalized"})).unwrap() + } + #[tokio::test] + async fn paginates_2500_signatures_and_preserves_boundaries() { + let mut offset = 0; + let mut calls = 0; + let (rows, exhausted) = signatures(3000, None, None, false, |config| { + calls += 1; + assert_eq!(config.limit, Some(1000)); + if offset > 0 { + assert_eq!( + config.before.unwrap().to_string(), + info(offset - 1).signature + ); + } + let end = (offset + 1000).min(2500); + let page = (offset..end).map(info).collect(); + offset = end; + std::future::ready(Ok(page)) + }) + .await + .unwrap(); + assert_eq!(calls, 3); + assert_eq!(rows.len(), 2500); + assert!(exhausted); + } + #[tokio::test] + async fn incremental_overflow_fails_instead_of_skipping_history() { + let result = signatures(2, None, Some(Signature::default()), true, |config| { + std::future::ready(Ok((1..=config.limit.unwrap()).map(info).collect())) + }) + .await; + assert!(result + .unwrap_err() + .to_string() + .contains("increase ingest_limit")); + } + #[tokio::test] + async fn rejects_a_provider_that_repeats_a_page() { + let result = signatures(2000, None, None, false, |_| { + std::future::ready(Ok((1..=1000).map(info).collect())) + }) + .await; + assert!(result.unwrap_err().to_string().contains("repeated")); + } +} diff --git a/adapters/src/solana_parser.rs b/adapters/src/solana_parser.rs index 3ef554a..0038608 100644 --- a/adapters/src/solana_parser.rs +++ b/adapters/src/solana_parser.rs @@ -245,62 +245,68 @@ pub fn extract_solana_token_transfers( let mut group_counts: std::collections::HashMap<(String, String, String), i32> = std::collections::HashMap::new(); - if let OptionSerializer::Some(pre_token_balances) = &meta.pre_token_balances { - if let OptionSerializer::Some(post_token_balances) = &meta.post_token_balances { - for post in post_token_balances { - let owner = match &post.owner { - OptionSerializer::Some(o) => o.clone(), - _ => continue, - }; - - let mint = post.mint.clone(); - let decimals = u32::from(post.ui_token_amount.decimals); - - let pre_raw = pre_token_balances - .iter() - .find(|p| p.account_index == post.account_index) - .map(|p| parse_token_amount_or_zero(&p.ui_token_amount.amount)) - .unwrap_or(0); - - let post_raw = parse_token_amount_or_zero(&post.ui_token_amount.amount); - let delta_raw = token_delta_raw(post_raw, pre_raw); - - if delta_raw == 0 { - continue; - } - - let amount = token_raw_to_decimal(token_delta_magnitude(delta_raw), decimals); - let symbol = spl_token_symbol(&mint); - - let (from, to) = if delta_raw > 0 { - // Incoming: unknown sender -> owner - ("unknown".to_string(), owner) - } else { - // Outgoing: owner -> unknown receiver - (owner, "unknown".to_string()) - }; - - let key = (from.clone(), to.clone(), mint.clone()); - let idx = group_counts.entry(key).or_insert(0); - let transfer_index = *idx; - *idx += 1; - - transfers.push(TokenTransfer { - id: Uuid::new_v4(), - raw_transaction_id: raw_tx_id, - network: network.to_string(), - token_address: mint, - token_symbol: Some(symbol), - from_address: from, - to_address: to, - amount, - decimals: token_decimals_i32(decimals), - transfer_index, - dataset_version_id: None, - created_at: Utc::now(), - }); - } + let pre = match &meta.pre_token_balances { + OptionSerializer::Some(v) => v.as_slice(), + _ => &[], + }; + let post = match &meta.post_token_balances { + OptionSerializer::Some(v) => v.as_slice(), + _ => &[], + }; + let mut seen = std::collections::HashSet::new(); + // Union both sides: closed accounts disappear from postTokenBalances; + // ownership changes must debit the former owner and credit the new owner. + for entry in post.iter().chain(pre) { + let owner = match &entry.owner { + OptionSerializer::Some(owner) => owner, + _ => continue, + }; + if !seen.insert((entry.account_index, &entry.mint, owner)) { + continue; } + let matches = |p: &&solana_transaction_status::UiTransactionTokenBalance| { + p.account_index == entry.account_index + && p.mint == entry.mint + && matches!(&p.owner, OptionSerializer::Some(o) if o == owner) + }; + let pre_raw = pre + .iter() + .find(matches) + .map(|p| parse_token_amount_or_zero(&p.ui_token_amount.amount)) + .unwrap_or(0); + let post_raw = post + .iter() + .find(matches) + .map(|p| parse_token_amount_or_zero(&p.ui_token_amount.amount)) + .unwrap_or(0); + let delta = token_delta_raw(post_raw, pre_raw); + if delta == 0 { + continue; + } + let decimals = u32::from(entry.ui_token_amount.decimals); + let (from, to) = if delta > 0 { + ("unknown".to_string(), owner.clone()) + } else { + (owner.clone(), "unknown".to_string()) + }; + let index = group_counts + .entry((from.clone(), to.clone(), entry.mint.clone())) + .or_default(); + transfers.push(TokenTransfer { + id: Uuid::new_v4(), + raw_transaction_id: raw_tx_id, + network: network.into(), + token_address: entry.mint.clone(), + token_symbol: Some(spl_token_symbol(&entry.mint)), + from_address: from, + to_address: to, + amount: token_raw_to_decimal(token_delta_magnitude(delta), decimals), + decimals: token_decimals_i32(decimals), + transfer_index: *index, + dataset_version_id: None, + created_at: Utc::now(), + }); + *index += 1; } transfers @@ -341,7 +347,14 @@ pub fn extract_solana_native_balance_deltas( solana_transaction_status::UiMessage::Parsed(msg) => { msg.account_keys.iter().map(|k| k.pubkey.clone()).collect() } - solana_transaction_status::UiMessage::Raw(msg) => msg.account_keys.clone(), + solana_transaction_status::UiMessage::Raw(msg) => { + let mut keys = msg.account_keys.clone(); + if let OptionSerializer::Some(loaded) = &meta.loaded_addresses { + keys.extend(loaded.writable.iter().cloned()); + keys.extend(loaded.readonly.iter().cloned()); + } + keys + } } } else { return vec![]; @@ -804,6 +817,39 @@ mod tests { assert_eq!(change, 1_000_000_000i128); } + #[test] + fn v0_loaded_accounts_and_closed_token_accounts_are_not_lost() { + let mut raw = serde_json::json!({ + "slot": 1, "blockTime": 1, "version": 0, + "transaction": {"signatures": ["sig"], "message": { + "header": {"numRequiredSignatures":1,"numReadonlySignedAccounts":0,"numReadonlyUnsignedAccounts":0}, + "accountKeys":["payer"], "recentBlockhash":"block", "instructions":[], "addressTableLookups":[] + }}, + "meta": {"err":null, "status":{"Ok":null}, "fee":0, + "preBalances":[100,0], "postBalances":[90,10], + "loadedAddresses":{"writable":["loaded"], "readonly":[]}, + "preTokenBalances":[{"accountIndex":1,"mint":"mint","owner":"owner", + "uiTokenAmount":{"amount":"100","decimals":6,"uiAmount":0.0001,"uiAmountString":"0.0001"}}], + "postTokenBalances":[] + } + }); + let _: EncodedConfirmedTransactionWithStatusMeta = + serde_json::from_value(raw.clone()).unwrap(); + let deltas = extract_solana_native_balance_deltas(None, "solana-mainnet", &raw); + assert_eq!(deltas.len(), 2); + assert_eq!(deltas[1].account_address, "loaded"); + assert_eq!(deltas[1].delta, lamports_to_sol(10)); + let closed = extract_solana_token_transfers(None, "solana-mainnet", &raw); + assert_eq!(closed.len(), 1); + assert_eq!(closed[0].from_address, "owner"); + raw["meta"]["postTokenBalances"] = raw["meta"]["preTokenBalances"].clone(); + raw["meta"]["postTokenBalances"][0]["owner"] = serde_json::json!("new-owner"); + let changed = extract_solana_token_transfers(None, "solana-mainnet", &raw); + assert_eq!(changed.len(), 2); + assert!(changed.iter().any(|t| t.to_address == "new-owner")); + assert!(changed.iter().any(|t| t.from_address == "owner")); + } + #[test] fn solana_materializer_contract() { let m = SolanaLedgerMaterializer; diff --git a/adapters/src/v2_repo.rs b/adapters/src/v2_repo.rs index 2467057..7d63044 100644 --- a/adapters/src/v2_repo.rs +++ b/adapters/src/v2_repo.rs @@ -1241,6 +1241,7 @@ pub fn build_dataset_completeness_upsert( #[derive(Debug)] #[allow(clippy::large_enum_variant)] pub enum ExportRecordBatch { + RawTransactions(Vec), TokenTransfers(Vec), NativeBalanceDeltas(Vec), DecodedEvents(Vec), @@ -1261,6 +1262,7 @@ impl ExportRecordBatch { /// exhausted. pub fn len(&self) -> usize { match self { + ExportRecordBatch::RawTransactions(v) => v.len(), ExportRecordBatch::TokenTransfers(v) => v.len(), ExportRecordBatch::NativeBalanceDeltas(v) => v.len(), ExportRecordBatch::DecodedEvents(v) => v.len(), @@ -1479,12 +1481,18 @@ pub fn build_dataset_filter_query( let mut n: usize = 0; let mut wheres: Vec = Vec::new(); + let is_bronze = table_name == "raw_transactions"; if target_id.is_some() { - sql.push_str(" JOIN target_matches tm ON tm.raw_transaction_id = dt.raw_transaction_id"); + sql.push_str(if is_bronze { + " JOIN target_matches tm ON tm.raw_transaction_id = dt.id" + } else { + " JOIN target_matches tm ON tm.raw_transaction_id = dt.raw_transaction_id" + }); } - if time_start.is_some() || time_end.is_some() { + if !is_bronze && (time_start.is_some() || time_end.is_some()) { sql.push_str(" JOIN raw_transactions rt ON rt.id = dt.raw_transaction_id"); } + let time_alias = if is_bronze { "dt" } else { "rt" }; if let Some(tid) = target_id { n += 1; @@ -1501,13 +1509,13 @@ pub fn build_dataset_filter_query( } if let Some(start) = time_start { n += 1; - wheres.push(format!("rt.timestamp >= ${n}")); + wheres.push(format!("{time_alias}.timestamp >= ${n}")); use sqlx::Arguments; args.add(start).map_err(|e| anyhow::anyhow!("{e}"))?; } if let Some(end) = time_end { n += 1; - wheres.push(format!("rt.timestamp <= ${n}")); + wheres.push(format!("{time_alias}.timestamp <= ${n}")); use sqlx::Arguments; args.add(end).map_err(|e| anyhow::anyhow!("{e}"))?; } @@ -2292,10 +2300,13 @@ impl Repository { // ----------------------------------------------------------------------- pub async fn save_raw_transactions(&self, txs: &[RawTransaction]) -> anyhow::Result<()> { + let mut tx = self.pool().begin().await?; for chunk in txs.chunks(Self::V2_BATCH_SIZE) { let (query, args) = build_raw_transaction_insert(chunk)?; - sqlx::query_with(&query, args).execute(self.pool()).await?; + sqlx::query_with(&query, args).execute(&mut *tx).await?; + Self::link_run_inputs(&mut tx, chunk).await?; } + tx.commit().await?; Ok(()) } @@ -2312,20 +2323,39 @@ impl Repository { &self, txs: &[RawTransaction], ) -> anyhow::Result> { + let mut tx = self.pool().begin().await?; let mut all_ids = Vec::with_capacity(txs.len()); for chunk in txs.chunks(Self::V2_BATCH_SIZE) { - let (query, args) = build_raw_transaction_upsert_returning(chunk)?; - let rows = sqlx::query_with(&query, args) - .fetch_all(self.pool()) - .await?; - for row in &rows { - let id: Uuid = row.try_get("id")?; - all_ids.push(id); - } + all_ids.extend(Self::upsert_raw_batch_in_tx(&mut tx, chunk).await?); } + tx.commit().await?; Ok(all_ids) } + pub async fn query_raw_transactions( + &self, + target_id: Option, + network: Option<&str>, + time_start: Option, + time_end: Option, + limit: i64, + offset: i64, + ) -> anyhow::Result> { + let (sql, args) = build_dataset_filter_query( + "dt.*", + "raw_transactions", + "dt.timestamp", + target_id, + network, + time_start, + time_end, + limit, + offset, + )?; + let rows = sqlx::query_with(&sql, args).fetch_all(self.pool()).await?; + rows.iter().map(row_to_raw_transaction).collect() + } + pub async fn get_raw_transaction_by_hash( &self, network: &str, @@ -2350,7 +2380,9 @@ impl Repository { let rows = sqlx::query( "SELECT id, network, tx_hash, timestamp, block_number, raw_metadata, \ source, ingestion_run_id, ingested_at \ - FROM raw_transactions WHERE ingestion_run_id = $1 ORDER BY timestamp", + FROM raw_transactions rt WHERE ingestion_run_id = $1 OR EXISTS (\ + SELECT 1 FROM ingestion_run_transactions i WHERE i.ingestion_run_id = $1 \ + AND i.raw_transaction_id = rt.id) ORDER BY timestamp, id", ) .bind(run_id) .fetch_all(self.pool()) @@ -3252,6 +3284,44 @@ impl Repository { rows.iter().map(row_to_decoded_event).collect() } + /// Complete indexed input set for one public wallet, independent of batch/run boundaries. + pub async fn hl_wallet_inputs( + &self, + wallet: &str, + network: &str, + ) -> anyhow::Result<(Vec, Vec)> { + let predicate = "dt.network = $1 AND EXISTS (SELECT 1 FROM target_matches tm JOIN index_targets t ON t.id = tm.target_id WHERE tm.raw_transaction_id = dt.raw_transaction_id AND t.network = $1 AND lower(t.address) = lower($2))"; + let fills = sqlx::query(&format!("SELECT dt.* FROM hl_fill_records dt WHERE {predicate} ORDER BY dt.fill_time, dt.id LIMIT 100001")) + .bind(network).bind(wallet).fetch_all(self.pool()).await?; + let funding = sqlx::query(&format!("SELECT dt.* FROM hl_funding_payments dt WHERE {predicate} ORDER BY dt.payment_time, dt.id LIMIT 100001")) + .bind(network).bind(wallet).fetch_all(self.pool()).await?; + anyhow::ensure!( + fills.len() <= 100000 && funding.len() <= 100000, + "HL analytics exceeds 100000 indexed events; refusing truncated calculation" + ); + Ok(( + fills + .iter() + .map(row_to_hl_fill_record) + .collect::>()?, + funding + .iter() + .map(row_to_hl_funding_payment) + .collect::>()?, + )) + } + + pub async fn canonical_decoded_events( + &self, + raw_ids: &[Uuid], + ) -> anyhow::Result> { + let rows = sqlx::query("SELECT * FROM decoded_events WHERE raw_transaction_id = ANY($1)") + .bind(raw_ids) + .fetch_all(self.pool()) + .await?; + rows.iter().map(row_to_decoded_event).collect() + } + /// Query Hyperliquid fill records with optional target, network, and time-window filters. #[allow(clippy::too_many_arguments)] pub async fn query_hl_fill_records( @@ -3530,12 +3600,33 @@ impl Repository { // Gold-tier: wallet_ledger and balance_history (P5-W1) // ----------------------------------------------------------------------- - /// Upsert wallet_ledger records. Uses ON CONFLICT DO NOTHING on the primary key - /// to maintain idempotency. pub async fn save_wallet_ledger_records( &self, records: &[WalletLedgerRecord], ) -> anyhow::Result<()> { + self.write_wallet_ledger_records(records, None).await + } + + pub async fn replace_wallet_ledger_records( + &self, + wallet: &str, + inputs: &[Uuid], + records: &[WalletLedgerRecord], + ) -> anyhow::Result<()> { + self.write_wallet_ledger_records(records, Some((wallet, inputs))) + .await + } + + async fn write_wallet_ledger_records( + &self, + records: &[WalletLedgerRecord], + replace: Option<(&str, &[Uuid])>, + ) -> anyhow::Result<()> { + let mut tx = self.pool().begin().await?; + if let Some((wallet, inputs)) = replace { + sqlx::query("DELETE FROM wallet_ledger WHERE wallet_address = $1 AND raw_transaction_id = ANY($2)") + .bind(wallet).bind(inputs).execute(&mut *tx).await?; + } for chunk in records.chunks(500) { let mut query_builder: sqlx::QueryBuilder = sqlx::QueryBuilder::new( "INSERT INTO wallet_ledger (id, raw_transaction_id, wallet_address, network, tx_hash, \ @@ -3570,17 +3661,28 @@ impl Repository { proceeds = EXCLUDED.proceeds, \ dataset_version_id = EXCLUDED.dataset_version_id", ); - query_builder.build().execute(self.pool()).await?; + query_builder.build().execute(&mut *tx).await?; } + tx.commit().await?; Ok(()) } /// Upsert balance_history records. pub async fn save_balance_snapshots(&self, records: &[BalanceSnapshot]) -> anyhow::Result<()> { + self.save_balance_snapshots_with_deltas(records, &Default::default()) + .await + } + + pub async fn save_balance_snapshots_with_deltas( + &self, + records: &[BalanceSnapshot], + deltas: &std::collections::HashMap, + ) -> anyhow::Result<()> { + let mut tx = self.pool().begin().await?; for chunk in records.chunks(500) { let mut query_builder: sqlx::QueryBuilder = sqlx::QueryBuilder::new( "INSERT INTO balance_history (id, wallet_address, asset_symbol, network, \ - timestamp, balance, tx_hash, dataset_version_id, created_at) ", + timestamp, balance, tx_hash, dataset_version_id, created_at, delta) ", ); query_builder.push_values(chunk, |mut b, r| { b.push_bind(r.id) @@ -3591,15 +3693,18 @@ impl Repository { .push_bind(&r.balance) .push_bind(&r.tx_hash) .push_bind(r.dataset_version_id) - .push_bind(r.created_at); + .push_bind(r.created_at) + .push_bind(deltas.get(&r.id)); }); query_builder.push( " ON CONFLICT (id) DO UPDATE SET \ balance = EXCLUDED.balance, \ + delta = COALESCE(EXCLUDED.delta, balance_history.delta), \ dataset_version_id = EXCLUDED.dataset_version_id", ); - query_builder.build().execute(self.pool()).await?; + query_builder.build().execute(&mut *tx).await?; } + tx.commit().await?; Ok(()) } @@ -3745,6 +3850,27 @@ impl Repository { .await } + async fn direct_wallet_scope( + &self, + target_id: Option, + network: Option<&str>, + ) -> anyhow::Result, Option)>> { + if let Some(id) = target_id { + let Some(target) = self.get_index_target(id).await? else { + return Ok(None); + }; + if target.kind != TargetKind::Wallet || network.is_some_and(|n| n != target.network) { + return Ok(None); + } + let Some(address) = target.address.filter(|a| !a.is_empty()) else { + return Ok(None); + }; + Ok(Some((Some(address), Some(target.network)))) + } else { + Ok(Some((None, network.map(str::to_owned)))) + } + } + // -- P5-W2: Hyperliquid Gold analytics query/export methods -- pub async fn query_hl_pnl_summary( @@ -3756,17 +3882,12 @@ impl Repository { limit: i64, offset: i64, ) -> anyhow::Result> { - let target_address = if let Some(tid) = target_id { - match self.get_index_target(tid).await? { - Some(t) => match t.address { - Some(addr) if !addr.is_empty() => Some(addr), - _ => return Ok(Vec::new()), - }, - None => return Ok(Vec::new()), - } - } else { - None + let Some((target_address, scoped_network)) = + self.direct_wallet_scope(target_id, network).await? + else { + return Ok(Vec::new()); }; + let network = scoped_network.as_deref(); let cols = "dt.id, dt.wallet_address, dt.coin, dt.network, dt.period_start, \ dt.period_end, dt.total_closed_pnl, dt.total_funding, dt.total_fees, \ dt.net_pnl, dt.trade_count, dt.fill_count, dt.avg_trade_size, \ @@ -3815,17 +3936,12 @@ impl Repository { limit: i64, offset: i64, ) -> anyhow::Result> { - let target_address = if let Some(tid) = target_id { - match self.get_index_target(tid).await? { - Some(t) => match t.address { - Some(addr) if !addr.is_empty() => Some(addr), - _ => return Ok(Vec::new()), - }, - None => return Ok(Vec::new()), - } - } else { - None + let Some((target_address, scoped_network)) = + self.direct_wallet_scope(target_id, network).await? + else { + return Ok(Vec::new()); }; + let network = scoped_network.as_deref(); let cols = "dt.id, dt.wallet_address, dt.coin, dt.network, dt.side, dt.entry_price, \ dt.exit_price, dt.size, dt.opened_at, dt.closed_at, dt.realized_pnl, \ dt.fees, dt.num_fills, dt.dataset_version_id, dt.created_at"; @@ -3995,6 +4111,32 @@ impl Repository { /// Bulk insert HL PnL summary records. pub async fn save_hl_pnl_summary(&self, records: &[HlPnlSummary]) -> anyhow::Result<()> { + self.write_hl_pnl_summary(records, None).await + } + + pub async fn replace_hl_pnl_summary( + &self, + wallet: &str, + network: &str, + records: &[HlPnlSummary], + ) -> anyhow::Result<()> { + self.write_hl_pnl_summary(records, Some((wallet, network))) + .await + } + + async fn write_hl_pnl_summary( + &self, + records: &[HlPnlSummary], + replace: Option<(&str, &str)>, + ) -> anyhow::Result<()> { + let mut tx = self.pool().begin().await?; + if let Some((wallet, network)) = replace { + sqlx::query("DELETE FROM hl_pnl_summary WHERE wallet_address = $1 AND network = $2") + .bind(wallet) + .bind(network) + .execute(&mut *tx) + .await?; + } for chunk in records.chunks(500) { let mut query_builder: sqlx::QueryBuilder = sqlx::QueryBuilder::new( "INSERT INTO hl_pnl_summary (id, wallet_address, coin, network, period_start, \ @@ -4021,13 +4163,40 @@ impl Repository { .push_bind(r.created_at); }); query_builder.push(HL_PNL_SUMMARY_CONFLICT_SQL); - query_builder.build().execute(self.pool()).await?; + query_builder.build().execute(&mut *tx).await?; } + tx.commit().await?; Ok(()) } /// Bulk insert HL trade history records. pub async fn save_hl_trade_history(&self, records: &[HlTradeHistory]) -> anyhow::Result<()> { + self.write_hl_trade_history(records, None).await + } + + pub async fn replace_hl_trade_history( + &self, + wallet: &str, + network: &str, + records: &[HlTradeHistory], + ) -> anyhow::Result<()> { + self.write_hl_trade_history(records, Some((wallet, network))) + .await + } + + async fn write_hl_trade_history( + &self, + records: &[HlTradeHistory], + replace: Option<(&str, &str)>, + ) -> anyhow::Result<()> { + let mut tx = self.pool().begin().await?; + if let Some((wallet, network)) = replace { + sqlx::query("DELETE FROM hl_trade_history WHERE wallet_address = $1 AND network = $2") + .bind(wallet) + .bind(network) + .execute(&mut *tx) + .await?; + } for chunk in records.chunks(500) { let mut query_builder: sqlx::QueryBuilder = sqlx::QueryBuilder::new( "INSERT INTO hl_trade_history (id, wallet_address, coin, network, side, \ @@ -4052,8 +4221,9 @@ impl Repository { .push_bind(r.created_at); }); query_builder.push(HL_TRADE_HISTORY_CONFLICT_SQL); - query_builder.build().execute(self.pool()).await?; + query_builder.build().execute(&mut *tx).await?; } + tx.commit().await?; Ok(()) } @@ -4196,6 +4366,21 @@ impl Repository { rows.iter().map(row_to_balance_snapshot).collect() } + pub async fn balance_snapshot_deltas( + &self, + ids: &[Uuid], + ) -> anyhow::Result> { + let rows = sqlx::query( + "SELECT id, delta FROM balance_history WHERE id = ANY($1) AND delta IS NOT NULL", + ) + .bind(ids) + .fetch_all(self.pool()) + .await?; + rows.iter() + .map(|row| Ok((row.try_get("id")?, row.try_get("delta")?))) + .collect() + } + // ----------------------------------------------------------------------- // Snapshotted streaming export (fixes PR #238 [P1] correctness) // ----------------------------------------------------------------------- @@ -4264,6 +4449,25 @@ impl Repository { .await?; let total = match dataset { + "raw_transactions" => { + stream_paged_in_tx( + &mut tx, + "dt.*", + "raw_transactions", + "dt.timestamp", + target_id, + network, + time_start, + time_end, + page_size, + hard_cap, + &cancel, + &tx_out, + row_to_raw_transaction, + ExportRecordBatch::RawTransactions, + ) + .await? + } "token_transfers" => { let cols = "dt.id, dt.raw_transaction_id, dt.network, dt.token_address, \ dt.token_symbol, dt.from_address, dt.to_address, dt.amount, \ diff --git a/adapters/src/workbench_repo.rs b/adapters/src/workbench_repo.rs new file mode 100644 index 0000000..6eef81f --- /dev/null +++ b/adapters/src/workbench_repo.rs @@ -0,0 +1,163 @@ +use crate::repo::Repository; +use crate::v2_repo::{ + build_ingestion_run_insert, build_raw_transaction_upsert_returning, build_target_match_insert, +}; +use spectraplex_core::v2::{IndexTarget, IngestionRun, RawTransaction, TargetKind, TargetMatch}; +use sqlx::{Postgres, Row, Transaction}; +use uuid::Uuid; + +impl Repository { + pub(crate) async fn upsert_raw_batch_in_tx( + tx: &mut Transaction<'_, Postgres>, + records: &[RawTransaction], + ) -> anyhow::Result> { + let (query, args) = build_raw_transaction_upsert_returning(records)?; + let rows = sqlx::query_with(&query, args).fetch_all(&mut **tx).await?; + Self::link_run_inputs(tx, records).await?; + rows.iter().map(|r| Ok(r.try_get("id")?)).collect() + } + + pub(crate) async fn link_run_inputs( + tx: &mut Transaction<'_, Postgres>, + records: &[RawTransaction], + ) -> anyhow::Result<()> { + let with_run: Vec<_> = records + .iter() + .filter(|r| r.ingestion_run_id.is_some()) + .collect(); + if with_run.is_empty() { + return Ok(()); + } + let networks: Vec<_> = with_run.iter().map(|r| r.network.as_str()).collect(); + let hashes: Vec<_> = with_run.iter().map(|r| r.tx_hash.as_str()).collect(); + let runs: Vec<_> = with_run.iter().filter_map(|r| r.ingestion_run_id).collect(); + sqlx::query("INSERT INTO ingestion_run_transactions (ingestion_run_id, raw_transaction_id) + SELECT i.run, r.id FROM unnest($1::text[], $2::text[], $3::uuid[]) AS i(network, hash, run) + JOIN raw_transactions r ON r.network = i.network AND r.tx_hash = i.hash + ON CONFLICT DO NOTHING") + .bind(networks).bind(hashes).bind(runs).execute(&mut **tx).await?; + Ok(()) + } + + /// Raw rows, run membership, target lineage and queued ETL commit together. + pub async fn import_raw_records( + &self, + target: &IndexTarget, + run: &IngestionRun, + records: &[RawTransaction], + ) -> anyhow::Result> { + let mut tx = self.pool().begin().await?; + let (query, args) = build_ingestion_run_insert(run)?; + sqlx::query_with(&query, args).execute(&mut *tx).await?; + for chunk in records.chunks(500) { + let ids = Self::upsert_raw_batch_in_tx(&mut tx, chunk).await?; + let matches: Vec<_> = ids + .iter() + .map(|id| TargetMatch { + id: Uuid::new_v4(), + target_id: target.id, + raw_transaction_id: *id, + match_reason: Some("import".to_string()), + matched_at: chrono::Utc::now(), + }) + .collect(); + let (query, args) = build_target_match_insert(&matches)?; + sqlx::query_with(&query, args).execute(&mut *tx).await?; + } + let materialization_id = if target.kind == TargetKind::Wallet { + let id = Uuid::new_v4(); + let scope = serde_json::json!({"wallet": target.address, "network": target.network, + "target_id": target.id, "ingestion_run_id": run.id}); + sqlx::query("INSERT INTO materialization_runs (id, dataset_name, scope) VALUES ($1, 'normalize', $2)") + .bind(id).bind(scope).execute(&mut *tx).await?; + Some(id) + } else { + None + }; + tx.commit().await?; + Ok(materialization_id) + } + + /// Lease-fenced commit: inputs, lineage, checkpoint and downstream work are + /// durable together. A crash cannot advance a cursor past unmaterialized data. + pub async fn commit_connector_batch( + &self, + target: &IndexTarget, + run_id: Uuid, + records: &[RawTransaction], + checkpoint: Option<&spectraplex_core::v2::Checkpoint>, + lease: (Uuid, &str), + ) -> anyhow::Result<()> { + let mut tx = self.pool().begin().await?; + let valid = sqlx::query_scalar::<_, Uuid>("SELECT j.id FROM ingestion_jobs j WHERE j.id = $1 AND j.status IN ('claimed', 'running') AND EXISTS (SELECT 1 FROM ingestion_job_attempts a WHERE a.job_id = j.id AND a.worker_id = $2 AND a.finished_at IS NULL) FOR UPDATE OF j") + .bind(lease.0).bind(lease.1).fetch_optional(&mut *tx).await?; + anyhow::ensure!(valid.is_some(), "Ingestion lease lost before commit"); + for chunk in records.chunks(500) { + let ids = Self::upsert_raw_batch_in_tx(&mut tx, chunk).await?; + let matches: Vec<_> = ids + .into_iter() + .map(|id| TargetMatch { + id: Uuid::new_v4(), + target_id: target.id, + raw_transaction_id: id, + match_reason: Some("provider".into()), + matched_at: chrono::Utc::now(), + }) + .collect(); + let (query, args) = build_target_match_insert(&matches)?; + sqlx::query_with(&query, args).execute(&mut *tx).await?; + } + if let Some(checkpoint) = checkpoint { + let (query, args) = crate::v2_repo::build_checkpoint_upsert(checkpoint)?; + sqlx::query_with(&query, args).execute(&mut *tx).await?; + } + if target.kind == TargetKind::Wallet && !records.is_empty() { + let scope = serde_json::json!({"wallet": target.address, "network": target.network, "target_id": target.id, "ingestion_run_id": run_id}); + sqlx::query("INSERT INTO materialization_runs (id, dataset_name, scope) VALUES ($1, 'normalize', $2)") + .bind(Uuid::new_v4()).bind(scope).execute(&mut *tx).await?; + } + sqlx::query("UPDATE ingestion_runs SET status = 'completed', finished_at = NOW(), records_written = $2 WHERE id = $1") + .bind(run_id).bind(records.len() as i64).execute(&mut *tx).await?; + tx.commit().await?; + Ok(()) + } + + pub async fn list_workbench_jobs( + &self, + owner: Option, + target: Option, + limit: i64, + offset: i64, + ) -> anyhow::Result> { + let rows = sqlx::query( + "WITH jobs AS ( + SELECT j.id, 'ingest' AS kind, j.status AS state, j.network, j.target_id, + NULL::text AS dataset, NULL::text AS format, j.error_message AS message, + j.created_at, j.updated_at, NULL::bigint AS record_count, + NULL::uuid AS ingestion_run_id, t.address AS wallet, j.mode::text AS mode + FROM ingestion_jobs j JOIN index_targets t ON t.id = j.target_id + WHERE ($1::uuid IS NULL OR t.owner_id = $1) + UNION ALL + SELECT m.id, 'materialize', m.status, r.network, r.target_id, m.dataset_name, NULL, + m.error_message, m.created_at, m.updated_at, m.output_record_count, r.id, t.address, NULL::text + FROM materialization_runs m + JOIN ingestion_runs r ON r.id::text = m.scope->>'ingestion_run_id' + JOIN index_targets t ON t.id = r.target_id + WHERE ($1::uuid IS NULL OR t.owner_id = $1) + UNION ALL + SELECT e.id, 'export', e.status, e.filters->>'network', t.id, e.dataset, e.format, + e.error_message, e.created_at, e.updated_at, e.record_count::bigint, NULL::uuid, t.address, NULL::text + FROM export_jobs e LEFT JOIN index_targets t ON t.id::text = e.filters->>'target_id' + WHERE ($1::uuid IS NULL OR (e.owner_id = $1 AND t.owner_id = $1)) + ) SELECT to_jsonb(jobs) AS job FROM jobs WHERE ($2::uuid IS NULL OR target_id = $2) + ORDER BY created_at DESC, id DESC LIMIT $3 OFFSET $4", + ) + .bind(owner) + .bind(target) + .bind(limit.clamp(1, 1000)) + .bind(offset.max(0)) + .fetch_all(self.pool()) + .await?; + rows.iter().map(|r| Ok(r.try_get("job")?)).collect() + } +} diff --git a/adapters/tests/ingestion_compat_test.rs b/adapters/tests/ingestion_compat_test.rs index fe132d9..131ffec 100644 --- a/adapters/tests/ingestion_compat_test.rs +++ b/adapters/tests/ingestion_compat_test.rs @@ -65,6 +65,7 @@ async fn pg_is_available() -> bool { macro_rules! require_pg { () => { if !pg_is_available().await { + assert!(std::env::var_os("TEST_DATABASE_URL").is_none(), "Configured test PostgreSQL is unavailable; refusing to skip integration coverage"); eprintln!( "SKIPPED: PostgreSQL not available at {} — set TEST_DATABASE_URL or start PostgreSQL", base_url() diff --git a/adapters/tests/migration_test.rs b/adapters/tests/migration_test.rs index ca317dc..1856a33 100644 --- a/adapters/tests/migration_test.rs +++ b/adapters/tests/migration_test.rs @@ -50,6 +50,7 @@ async fn pg_is_available() -> bool { macro_rules! require_pg { () => { if !pg_is_available().await { + assert!(std::env::var_os("TEST_DATABASE_URL").is_none(), "Configured test PostgreSQL is unavailable; refusing to skip integration coverage"); eprintln!( "SKIPPED: PostgreSQL not available at {} — set TEST_DATABASE_URL or start PostgreSQL", base_url() @@ -492,7 +493,13 @@ async fn network_seed_data_13_networks() { let (pool, db_name) = create_test_db("nets").await; run_all_migrations(&pool).await; - assert_eq!(row_count(&pool, "networks").await, 13); + assert_eq!(row_count(&pool, "networks").await, 16); + let demo_count: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM networks WHERE id LIKE '%-demo' AND is_testnet") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(demo_count, 3); // Verify specific networks let expected = [ diff --git a/adapters/tests/operational_hardening_test.rs b/adapters/tests/operational_hardening_test.rs index 1c73abf..a1c6e8f 100644 --- a/adapters/tests/operational_hardening_test.rs +++ b/adapters/tests/operational_hardening_test.rs @@ -46,6 +46,7 @@ async fn pg_is_available() -> bool { macro_rules! require_pg { () => { if !pg_is_available().await { + assert!(std::env::var_os("TEST_DATABASE_URL").is_none(), "Configured test PostgreSQL is unavailable; refusing to skip integration coverage"); eprintln!( "SKIPPED: PostgreSQL not available at {} — set TEST_DATABASE_URL or start PostgreSQL", base_url() @@ -135,6 +136,129 @@ fn make_target( } } +#[tokio::test] +async fn connector_commit_is_atomic_and_lease_fenced() { + require_pg!(); + use spectraplex_adapters::v2_repo::EnqueueIngestionJobParams; + use spectraplex_core::v2::{ + Checkpoint, IngestionJobMode, IngestionJobStatus, IngestionRun, RawTransaction, + }; + let (repo, pool, db) = setup_test_repo("atomic_connector").await; + let target = repo + .create_index_target(&make_target( + TargetKind::Wallet, + ChainFamily::Evm, + "ethereum-mainnet", + Some("0x1111111111111111111111111111111111111111"), + None, + )) + .await + .unwrap(); + let job = repo + .enqueue_ingestion_job(&EnqueueIngestionJobParams { + target_id: Some(target.id), + network: &target.network, + mode: "incremental", + priority: 0, + idempotency_key: None, + requested_by: None, + callback_url: None, + }) + .await + .unwrap(); + repo.claim_ingestion_job("owner").await.unwrap().unwrap(); + let now = Utc::now(); + let run = IngestionRun { + id: Uuid::new_v4(), + target_id: Some(target.id), + network: target.network.clone(), + source: "rpc".into(), + mode: IngestionJobMode::Incremental, + status: IngestionJobStatus::Running, + started_at: now, + finished_at: None, + records_written: 0, + error_message: None, + cursor_state: None, + }; + repo.create_ingestion_run(&run).await.unwrap(); + let raw = RawTransaction { + id: Uuid::new_v4(), + network: target.network.clone(), + tx_hash: "test-atomic".into(), + timestamp: 1, + block_number: Some(1), + raw_metadata: serde_json::json!({}), + source: "rpc".into(), + ingestion_run_id: Some(run.id), + ingested_at: now, + }; + let cp = Checkpoint { + id: Uuid::new_v4(), + target_id: target.id, + network: target.network.clone(), + source: "rpc".into(), + cursor: serde_json::json!({"last_block":1}), + updated_at: now, + }; + sqlx::raw_sql("CREATE FUNCTION reject_materialization() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected outbox failure'; END $$; CREATE TRIGGER reject_materialization BEFORE INSERT ON materialization_runs FOR EACH ROW EXECUTE FUNCTION reject_materialization();").execute(&pool).await.unwrap(); + assert!(repo + .commit_connector_batch( + &target, + run.id, + std::slice::from_ref(&raw), + Some(&cp), + (job.id, "owner") + ) + .await + .is_err()); + for table in [ + "raw_transactions", + "target_matches", + "ingestion_run_transactions", + "checkpoints", + "materialization_runs", + ] { + let count: i64 = sqlx::query_scalar(&format!("SELECT count(*) FROM {table}")) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(count, 0, "partial commit in {table}"); + } + sqlx::query("DROP TRIGGER reject_materialization ON materialization_runs") + .execute(&pool) + .await + .unwrap(); + assert!(repo + .commit_connector_batch( + &target, + run.id, + std::slice::from_ref(&raw), + Some(&cp), + (job.id, "stale-owner") + ) + .await + .is_err()); + repo.commit_connector_batch(&target, run.id, &[raw], Some(&cp), (job.id, "owner")) + .await + .unwrap(); + for table in [ + "raw_transactions", + "target_matches", + "ingestion_run_transactions", + "checkpoints", + "materialization_runs", + ] { + let count: i64 = sqlx::query_scalar(&format!("SELECT count(*) FROM {table}")) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(count, 1, "missing durable write in {table}"); + } + pool.close().await; + drop_test_db(&db).await; +} + fn make_evm_tx_with_transfer(tx_hash: &str, wallet: &str) -> Transaction { let from_padded = format!("0x000000000000000000000000{}", &wallet[2..]); let to_padded = "0x0000000000000000000000002222222222222222222222222222222222222222"; diff --git a/api/Cargo.toml b/api/Cargo.toml index d229b32..c595109 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -1,7 +1,10 @@ [package] name = "spectraplex-api" -version = "0.1.0" -edition = "2021" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true [dependencies] sha2 = "0.10" diff --git a/api/src/export_csv.rs b/api/src/export_csv.rs index 2f6d86d..f96e1e0 100644 --- a/api/src/export_csv.rs +++ b/api/src/export_csv.rs @@ -25,9 +25,18 @@ use spectraplex_core::materializer::{ }; use std::io::Write; -/// Quote a CSV field if it contains any of `,`, `"`, or newline. +/// Escape textual fields and neutralize spreadsheet formula prefixes. +/// Numeric amounts are emitted separately and retain their numeric representation. pub(crate) fn csv_escape(s: &str) -> String { - if s.contains(',') || s.contains('"') || s.contains('\n') { + let protected; + let s = if s.trim_start().starts_with(['=', '+', '-', '@']) || s.starts_with(['\t', '\r', '\n']) + { + protected = format!("'{s}"); + protected.as_str() + } else { + s + }; + if s.contains([',', '"', '\n', '\r']) { format!("\"{}\"", s.replace('"', "\"\"")) } else { s.to_string() @@ -56,7 +65,7 @@ pub(crate) fn write_token_transfers_csv_rows( .unwrap_or_default(), csv_escape(&r.network), csv_escape(&r.token_address), - r.token_symbol.as_deref().unwrap_or(""), + csv_escape(r.token_symbol.as_deref().unwrap_or("")), csv_escape(&r.from_address), csv_escape(&r.to_address), r.amount, @@ -166,13 +175,13 @@ pub(crate) fn write_hl_fills_csv_rows( csv_escape(&r.side), r.price, r.size, - r.direction.as_deref().unwrap_or(""), + csv_escape(r.direction.as_deref().unwrap_or("")), r.closed_pnl .as_ref() .map(|v| v.to_string()) .unwrap_or_default(), r.fee.as_ref().map(|v| v.to_string()).unwrap_or_default(), - r.fee_token.as_deref().unwrap_or(""), + csv_escape(r.fee_token.as_deref().unwrap_or("")), r.fill_time, r.order_id.map(|v| v.to_string()).unwrap_or_default(), r.trade_id.map(|v| v.to_string()).unwrap_or_default(), @@ -247,7 +256,7 @@ pub(crate) fn write_hl_positions_csv_rows( csv_escape(&r.side), r.size_delta, r.price, - r.direction.as_deref().unwrap_or(""), + csv_escape(r.direction.as_deref().unwrap_or("")), csv_escape(&r.source_event), r.dataset_version_id .map(|u| u.to_string()) @@ -285,12 +294,12 @@ pub(crate) fn write_wallet_ledger_csv_rows( csv_escape(&r.entry_type), csv_escape(&r.asset_symbol), r.amount, - r.counterparty_address.as_deref().unwrap_or(""), + csv_escape(r.counterparty_address.as_deref().unwrap_or("")), r.fee_amount .as_ref() .map(|v| v.to_string()) .unwrap_or_default(), - r.fee_asset.as_deref().unwrap_or(""), + csv_escape(r.fee_asset.as_deref().unwrap_or("")), r.cost_basis .as_ref() .map(|v| v.to_string()) @@ -437,10 +446,10 @@ pub(crate) fn write_protocol_events_csv_rows( r.id, csv_escape(&r.network), csv_escape(&r.protocol_address), - r.protocol_name.as_deref().unwrap_or(""), + csv_escape(r.protocol_name.as_deref().unwrap_or("")), csv_escape(&r.event_type), csv_escape(&r.event_details.to_string()), - r.pool_address.as_deref().unwrap_or(""), + csv_escape(r.pool_address.as_deref().unwrap_or("")), r.raw_event_id.map(|u| u.to_string()).unwrap_or_default(), r.timestamp, r.dataset_version_id @@ -472,11 +481,11 @@ pub(crate) fn write_pool_snapshots_csv_rows( csv_escape(&r.network), csv_escape(&r.pool_address), csv_escape(&r.protocol_address), - r.protocol_name.as_deref().unwrap_or(""), + csv_escape(r.protocol_name.as_deref().unwrap_or("")), csv_escape(&r.token0_address), - r.token0_symbol.as_deref().unwrap_or(""), + csv_escape(r.token0_symbol.as_deref().unwrap_or("")), csv_escape(&r.token1_address), - r.token1_symbol.as_deref().unwrap_or(""), + csv_escape(r.token1_symbol.as_deref().unwrap_or("")), r.reserve0, r.reserve1, r.tvl_usd @@ -511,28 +520,42 @@ pub(crate) fn write_wallet_ledger_tax_csv_rows( .map(|dt| dt.format("%Y-%m-%d %H:%M:%S UTC").to_string()) .unwrap_or_else(|| r.timestamp.to_string()); - let (sent_asset, sent_amount, recv_asset, recv_amount) = if r.amount < BigDecimal::from(0) { - ( - r.asset_symbol.as_str(), - r.amount.abs().to_string(), - "", - String::new(), - ) + let (sent_asset, sent_amount, recv_asset, recv_amount) = + if r.entry_type == "fee" && r.amount < BigDecimal::from(0) { + ("", String::new(), "", String::new()) + } else if r.amount < BigDecimal::from(0) { + ( + r.asset_symbol.as_str(), + r.amount.abs().to_string(), + "", + String::new(), + ) + } else { + ( + "", + String::new(), + r.asset_symbol.as_str(), + r.amount.to_string(), + ) + }; + + // A dedicated fee row is a fee, not both a sent amount and a fee. + // Maker rebates are receipts, not a receipt plus a negative fee. + let (fee_asset, fee_amount) = if r.entry_type == "fee" { + if r.amount < BigDecimal::from(0) { + (r.asset_symbol.as_str(), r.amount.abs().to_string()) + } else { + ("", String::new()) + } } else { ( - "", - String::new(), - r.asset_symbol.as_str(), - r.amount.to_string(), + r.fee_asset.as_deref().unwrap_or(""), + r.fee_amount + .as_ref() + .map(ToString::to_string) + .unwrap_or_default(), ) }; - - let fee_asset = r.fee_asset.as_deref().unwrap_or(""); - let fee_amount = r - .fee_amount - .as_ref() - .map(|v| v.to_string()) - .unwrap_or_default(); let cost_basis = r .cost_basis .as_ref() @@ -569,3 +592,57 @@ pub(crate) fn write_wallet_ledger_tax_csv_rows( } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tax_csv_does_not_double_count_dedicated_fees_or_rebates() { + let mut record = WalletLedgerRecord { + id: uuid::Uuid::nil(), + raw_transaction_id: None, + wallet_address: "wallet".into(), + network: "network".into(), + tx_hash: "hash".into(), + timestamp: 1, + entry_type: "fee".into(), + asset_symbol: "USDC".into(), + amount: BigDecimal::from(-1), + counterparty_address: None, + fee_amount: Some(BigDecimal::from(1)), + fee_asset: Some("USDC".into()), + cost_basis: None, + proceeds: None, + dataset_version_id: None, + created_at: chrono::Utc::now(), + }; + let mut bytes = Vec::new(); + write_wallet_ledger_tax_csv_rows(std::slice::from_ref(&record), &mut bytes).unwrap(); + let text = String::from_utf8(bytes).unwrap(); + let fields: Vec<_> = text.trim_end().split(',').collect(); + assert_eq!(&fields[2..8], &["", "", "", "", "USDC", "1"]); + record.amount = BigDecimal::from(1); + record.fee_amount = Some(BigDecimal::from(-1)); + let mut bytes = Vec::new(); + write_wallet_ledger_tax_csv_rows(&[record], &mut bytes).unwrap(); + let text = String::from_utf8(bytes).unwrap(); + let fields: Vec<_> = text.trim_end().split(',').collect(); + assert_eq!(&fields[2..8], &["", "", "USDC", "1", "", ""]); + } + #[test] + fn textual_cells_cannot_execute_spreadsheet_formulas() { + for value in [ + "=1+1", "+SUM(1)", "-command", "@SUM(1)", " =cmd", "\tcmd", "\rcmd", + ] { + let escaped = csv_escape(value); + assert!( + escaped.starts_with('\'') || escaped.starts_with("\"'"), + "{escaped}" + ); + } + assert_eq!(csv_escape("a\rb"), "\"a\rb\""); + assert_eq!(csv_escape("a,\"b\""), "\"a,\"\"b\"\"\""); + assert_eq!(csv_escape("ETH"), "ETH"); + } +} diff --git a/api/src/export_stream.rs b/api/src/export_stream.rs index 445c4d9..7547f9c 100644 --- a/api/src/export_stream.rs +++ b/api/src/export_stream.rs @@ -210,6 +210,7 @@ pub(crate) async fn fetch_export_metadata( /// `None` if the dataset is unknown. fn csv_header_for(dataset: &str) -> Option<&'static str> { Some(match dataset { + "raw_transactions" => "id,network,tx_hash,timestamp,block_number,source,raw_metadata\n", "token_transfers" => token_transfers_csv_header(), "native_balance_deltas" => native_balance_deltas_csv_header(), "decoded_events" => decoded_events_csv_header(), @@ -244,6 +245,25 @@ fn encode_batch( buf: &mut Vec, ) -> anyhow::Result<()> { match (format, batch) { + (ExportFormat::Jsonl, ExportRecordBatch::RawTransactions(v)) => encode_jsonl_page(v, buf), + (ExportFormat::Csv, ExportRecordBatch::RawTransactions(v)) => { + use crate::export_csv::csv_escape; + use std::io::Write; + for row in v { + writeln!( + buf, + "{},{},{},{},{},{},{}", + row.id, + csv_escape(&row.network), + csv_escape(&row.tx_hash), + row.timestamp, + row.block_number.map(|v| v.to_string()).unwrap_or_default(), + csv_escape(&row.source), + csv_escape(&row.raw_metadata.to_string()) + )?; + } + Ok(()) + } (ExportFormat::Jsonl, ExportRecordBatch::TokenTransfers(v)) => encode_jsonl_page(v, buf), (ExportFormat::Jsonl, ExportRecordBatch::NativeBalanceDeltas(v)) => { encode_jsonl_page(v, buf) diff --git a/api/src/main.rs b/api/src/main.rs index 7b2e300..4279936 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -3,6 +3,8 @@ mod export_stream; mod export_worker; mod materialize_worker; mod stream_worker; +mod web; +mod workbench; mod worker; use axum::{ @@ -290,6 +292,11 @@ async fn main() -> anyhow::Result<()> { let config = AppConfig::load()?; config.validate()?; + anyhow::ensure!( + config.api_key.is_some(), + "Set SPECTRAPLEX_API_KEY (or run ./scripts/setup.sh) before starting the API" + ); + tokio::fs::create_dir_all(&config.export_dir).await?; tracing_subscriber::fmt() .with_env_filter( @@ -369,7 +376,44 @@ async fn main() -> anyhow::Result<()> { worker_cancel.clone(), ); + let app = app_router(shared_state); + let addr: SocketAddr = format!("{}:{}", config.host, config.port).parse()?; + info!(version = env!("CARGO_PKG_VERSION"), "Listening on {}", addr); + let listener = tokio::net::TcpListener::bind(addr).await?; + axum::serve(listener, app) + .with_graceful_shutdown(async move { + shutdown_signal().await; + info!("Shutdown signal received"); + worker_cancel.cancel(); + }) + .await?; + Ok(()) +} + +async fn shutdown_signal() { + #[cfg(unix)] + { + let mut terminate = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("install SIGTERM handler"); + tokio::select! { + _ = tokio::signal::ctrl_c() => {}, + _ = terminate.recv() => {}, + } + } + #[cfg(not(unix))] + let _ = tokio::signal::ctrl_c().await; +} + +fn app_router(shared_state: Arc) -> Router { let protected = Router::new() + .route("/v1/session", get(workbench::session)) + .route("/v1/jobs", get(workbench::jobs)) + .route("/v1/demo/{chain}", post(workbench::demo)) + .route( + "/v1/targets/{target_id}/import", + post(workbench::import_records), + ) .route("/v1/ingest", post(trigger_ingest)) .route("/v1/ingest/batch", post(trigger_batch_ingest)) .route("/v1/normalize", post(trigger_normalize)) @@ -433,8 +477,11 @@ async fn main() -> anyhow::Result<()> { require_auth, )); - let app = Router::new() + Router::new() + .route("/", get(web::index)) + .route("/assets/{name}", get(web::asset)) .route("/health", get(health_check)) + .route("/ready", get(readiness_check)) .merge(protected) .layer(axum::extract::DefaultBodyLimit::max(1_048_576)) .layer(TimeoutLayer::with_status_code( @@ -442,26 +489,26 @@ async fn main() -> anyhow::Result<()> { Duration::from_secs(60), )) .layer(TraceLayer::new_for_http()) - .with_state(shared_state); - - let addr: SocketAddr = format!("{}:{}", config.host, config.port).parse()?; - info!("Listening on {}", addr); - let listener = tokio::net::TcpListener::bind(addr).await?; - axum::serve(listener, app) - .with_graceful_shutdown(async move { - let _ = tokio::signal::ctrl_c().await; - info!("Shutdown signal received"); - worker_cancel.cancel(); - }) - .await?; - - Ok(()) + .layer(middleware::from_fn(web::security_headers)) + .with_state(shared_state) } async fn health_check() -> &'static str { "OK" } +async fn readiness_check( + State(state): State>, +) -> Result, AppError> { + tokio::time::timeout(Duration::from_secs(3), state.repo.check_connection()) + .await + .map_err(|_| AppError::service_unavailable("Database unavailable"))? + .map_err(|_| AppError::service_unavailable("Database unavailable"))?; + Ok(Json( + serde_json::json!({"status": "ready", "version": env!("CARGO_PKG_VERSION")}), + )) +} + async fn require_auth( State(state): State>, mut req: Request, @@ -668,6 +715,9 @@ struct ExportJobRequest { #[derive(Serialize)] struct DatasetInfo { name: String, + tier: String, + queryable: bool, + chain_families: Vec, latest_version: Option, latest_version_status: Option, } @@ -1898,11 +1948,7 @@ fn format_entry_type(et: &spectraplex_core::models::EntryType) -> &'static str { } fn csv_escape(s: &str) -> String { - if s.contains(',') || s.contains('"') || s.contains('\n') { - format!("\"{}\"", s.replace('"', "\"\"")) - } else { - s.to_string() - } + export_csv::csv_escape(s) } async fn export_ledger( @@ -2771,6 +2817,7 @@ async fn get_network( /// is kept for backward-compatible test assertions and `join()` in error /// messages. const QUERYABLE_DATASETS: &[&str] = &[ + "raw_transactions", "token_transfers", "native_balance_deltas", "decoded_events", @@ -2813,6 +2860,9 @@ async fn list_all_datasets( .map_err(AppError::internal)?; datasets.push(DatasetInfo { name: sql_name.to_string(), + tier: ds.tier().to_string(), + queryable: DatasetRegistry::is_queryable(sql_name), + chain_families: ds.chain_families().to_vec(), latest_version: latest.as_ref().map(|v| v.version), latest_version_status: latest.as_ref().map(|v| v.status.to_string()), }); @@ -2840,7 +2890,7 @@ async fn query_dataset_records( State(state): State>, Extension(owner): Extension, Path(name): Path, - Query(params): Query, + Query(mut params): Query, ) -> Result, AppError> { // Tenant isolation: require target_id and validate ownership. if let Some(_owner_id) = owner.0 { @@ -2858,11 +2908,42 @@ async fn query_dataset_records( ))); } validate_date_range(params.time_start, params.time_end)?; + if let Some(target_id) = params.target_id { + let target = state + .repo + .get_index_target(target_id) + .await + .map_err(AppError::internal)? + .ok_or_else(|| AppError::not_found("Target not found"))?; + if params + .network + .as_ref() + .is_some_and(|network| network != &target.network) + { + return Err(AppError::bad_request("network does not match target")); + } + params.network = Some(target.network); + } let limit = clamp_limit(params.limit); let offset = clamp_offset(params.offset); let net = params.network.as_deref(); let result = match name.as_str() { + "raw_transactions" => { + let records = state + .repo + .query_raw_transactions( + params.target_id, + net, + params.time_start, + params.time_end, + limit, + offset, + ) + .await + .map_err(AppError::internal)?; + serde_json::to_value(records).map_err(AppError::internal)? + } "token_transfers" => { let records = state .repo @@ -3058,6 +3139,7 @@ async fn query_dataset_records( /// Derived from `DatasetRegistry::exportable()` — the static `&[&str]` form /// is kept for backward-compatible test assertions. const EXPORTABLE_DATASETS: &[&str] = &[ + "raw_transactions", "token_transfers", "native_balance_deltas", "decoded_events", @@ -3348,10 +3430,17 @@ async fn get_export_job_status( } } +#[derive(Deserialize)] +struct DownloadParams { + #[serde(default)] + provenance: bool, +} + async fn download_export( State(state): State>, Extension(owner): Extension, Path(job_id): Path, + Query(params): Query, ) -> Result { let job = state .repo @@ -3372,13 +3461,24 @@ async fn download_export( .as_deref() .ok_or_else(|| AppError::internal("Export result location missing"))?; - let file_path = format!("{}/{}", state.config.export_dir, result_location); - let body = tokio::fs::read(&file_path) + let file_path = if params.provenance { + format!( + "{}/exports/{}.provenance.json", + state.config.export_dir, job_id + ) + } else { + format!("{}/{}", state.config.export_dir, result_location) + }; + let file = tokio::fs::File::open(&file_path) .await - .map_err(|e| AppError::internal(format!("Failed to read export file: {e}")))?; + .map_err(|e| AppError::internal(format!("Failed to open export file: {e}")))?; + let body = axum::body::Body::from_stream(tokio_util::io::ReaderStream::new(file)); - let format = job.format; - let ct = content_type_for_format(format); + let ct = if params.provenance { + "application/json" + } else { + content_type_for_format(job.format) + }; let sanitize = |s: &str| -> String { s.chars() @@ -3386,7 +3486,11 @@ async fn download_export( .collect() }; let safe_dataset = sanitize(job.dataset.as_sql_str()); - let safe_format = sanitize(&job.format.to_string()); + let safe_format = if params.provenance { + "provenance.json".into() + } else { + sanitize(&job.format.to_string()) + }; let disposition = format!( "attachment; filename=\"{}-{}.{}\"", safe_dataset, job_id, safe_format, @@ -3913,74 +4017,7 @@ mod tests { } fn test_router_with_state(state: Arc) -> Router { - let protected = Router::new() - .route("/v1/ingest", post(trigger_ingest)) - .route("/v1/ingest/batch", post(trigger_batch_ingest)) - .route("/v1/normalize", post(trigger_normalize)) - .route("/v1/jobs/{job_id}", get(get_job_status)) - .route("/v1/transactions/{wallet}", get(get_transactions)) - .route("/v1/ledger/{wallet}", get(get_ledger)) - .route("/v1/export/{wallet}", get(export_ledger)) - .route("/v1/balances/{wallet}", get(get_balances)) - .route( - "/v1/transactions/{wallet}/{tx_hash}", - get(get_single_transaction), - ) - .route("/v1/stats/{wallet}", get(get_wallet_stats)) - .route("/v1/stream/start", post(start_stream)) - .route("/v1/stream/{stream_id}/stop", post(stop_stream)) - .route("/v1/streams", get(list_streams)) - .route("/v1/targets", post(register_target)) - .route("/v1/targets", get(list_targets)) - .route("/v1/targets/{target_id}", get(get_target)) - .route( - "/v1/targets/{target_id}/ingest", - post(trigger_target_ingest), - ) - .route("/v1/api-keys", post(create_api_key_handler)) - .route("/v1/api-keys", get(list_api_keys_handler)) - .route("/v1/api-keys/{key_id}", delete(revoke_api_key_handler)) - .route("/v1/networks", get(list_networks)) - .route("/v1/networks/{network_id}", get(get_network)) - .route("/v1/datasets", get(list_all_datasets)) - .route( - "/v1/datasets/{name}/versions", - get(list_dataset_versions_handler), - ) - .route("/v1/datasets/{name}/records", get(query_dataset_records)) - .route( - "/v1/datasets/{name}/completeness", - get(get_dataset_completeness_handler), - ) - .route( - "/v1/datasets/{name}/status", - get(get_dataset_status_handler), - ) - .route("/v1/export/dataset", post(create_export_job)) - .route("/v1/export/jobs/{job_id}", get(get_export_job_status)) - .route("/v1/export/jobs/{job_id}/download", get(download_export)) - .route("/v1/export/tax", get(tax_export)) - .route("/v1/forensics/activity", get(forensics_activity_handler)) - .route("/v1/analytics/hl/trader", get(hl_trader_analytics_handler)) - .route("/v1/analytics/hl/market", get(hl_market_analytics_handler)) - .route( - "/v1/analytics/protocol/activity", - get(protocol_activity_handler), - ) - .route("/v1/analytics/protocol/tvl", get(protocol_tvl_handler)) - .layer(middleware::from_fn_with_state( - Arc::clone(&state), - rate_limit_middleware, - )) - .layer(middleware::from_fn_with_state( - Arc::clone(&state), - require_auth, - )); - - Router::new() - .route("/health", get(health_check)) - .merge(protected) - .with_state(state) + app_router(state) } #[test] @@ -6245,7 +6282,7 @@ mod tests { #[test] fn test_queryable_datasets_count() { - assert_eq!(QUERYABLE_DATASETS.len(), 12); + assert_eq!(QUERYABLE_DATASETS.len(), 13); } #[test] @@ -6294,6 +6331,9 @@ mod tests { fn test_dataset_info_serialization() { let info = DatasetInfo { name: "token_transfers".to_string(), + tier: "silver".into(), + queryable: true, + chain_families: vec![ChainFamily::Solana], latest_version: Some(1), latest_version_status: Some("active".to_string()), }; @@ -6307,6 +6347,9 @@ mod tests { fn test_dataset_info_serialization_no_version() { let info = DatasetInfo { name: "positions".to_string(), + tier: "silver".into(), + queryable: true, + chain_families: vec![ChainFamily::Hyperliquid], latest_version: None, latest_version_status: None, }; @@ -8067,7 +8110,7 @@ mod tests { #[test] fn test_exportable_datasets_includes_gold() { - assert_eq!(EXPORTABLE_DATASETS.len(), 12); + assert_eq!(EXPORTABLE_DATASETS.len(), 13); assert!(EXPORTABLE_DATASETS.contains(&"wallet_ledger")); assert!(EXPORTABLE_DATASETS.contains(&"balance_history")); assert!(EXPORTABLE_DATASETS.contains(&"hl_pnl_summary")); diff --git a/api/src/materialize_worker.rs b/api/src/materialize_worker.rs index a4ca164..df3c78c 100644 --- a/api/src/materialize_worker.rs +++ b/api/src/materialize_worker.rs @@ -502,8 +502,9 @@ pub(crate) async fn execute_normalize( run_id = %run_id, written = silver_result.total_written, failed = silver_result.total_failed, - "Bronze-native Silver materialization had partial failures" + "Bronze-native Silver/Gold materialization had partial failures" ); + anyhow::bail!("Materialization failed for {} records; retry normalization using this ingestion run", silver_result.total_failed); } return Ok(silver_result); @@ -546,7 +547,7 @@ fn dataset_completeness_for_ingestion_run( last_ingestion_run_id: Some(irun.id), records_count: usize_to_i64_or_max(*ds_count), gap_ranges: None, - notes: None, + notes: Some("Processing status covers this run's indexed inputs, not complete chain history or complete protocol decoding.".into()), created_at: now, updated_at: now, }) diff --git a/api/src/web.rs b/api/src/web.rs new file mode 100644 index 0000000..29f399d --- /dev/null +++ b/api/src/web.rs @@ -0,0 +1,53 @@ +use axum::{ + http::{header, StatusCode}, + response::IntoResponse, +}; + +pub(super) async fn index() -> impl IntoResponse { + ( + [(header::CONTENT_TYPE, "text/html; charset=utf-8")], + include_str!("../static/index.html"), + ) +} + +pub(super) async fn asset( + axum::extract::Path(name): axum::extract::Path, +) -> impl IntoResponse { + match name.as_str() { + "app.js" => ( + StatusCode::OK, + [(header::CONTENT_TYPE, "text/javascript; charset=utf-8")], + include_str!("../static/app.js"), + ), + "style.css" => ( + StatusCode::OK, + [(header::CONTENT_TYPE, "text/css; charset=utf-8")], + include_str!("../static/style.css"), + ), + "samples.json" => ( + StatusCode::OK, + [(header::CONTENT_TYPE, "application/json")], + include_str!("../static/samples.json"), + ), + _ => ( + StatusCode::NOT_FOUND, + [(header::CONTENT_TYPE, "text/plain")], + "Not found", + ), + } +} + +pub(super) async fn security_headers( + req: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + let mut response = next.run(req).await; + for (name, value) in [ + ("cache-control", "no-store"), + ("x-content-type-options", "nosniff"), + ("x-frame-options", "DENY"), + ("referrer-policy", "no-referrer"), + ("content-security-policy", "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; connect-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'"), + ] { response.headers_mut().insert(axum::http::HeaderName::from_static(name), axum::http::HeaderValue::from_static(value)); } + response +} diff --git a/api/src/workbench.rs b/api/src/workbench.rs new file mode 100644 index 0000000..a3b011f --- /dev/null +++ b/api/src/workbench.rs @@ -0,0 +1,267 @@ +use super::*; +use spectraplex_core::v2::{IngestionJobMode, IngestionRun, RawTransaction}; + +pub(super) async fn session( + State(state): State>, + Extension(owner): Extension, +) -> Json { + let networks: Vec<_> = state + .provider_registry + .enabled_networks() + .map(|id| id.to_string()) + .collect(); + Json( + serde_json::json!({"version": env!("CARGO_PKG_VERSION"), "admin": owner.0.is_none(), + "owner_id": owner.0, "configured_networks": networks}), + ) +} + +#[derive(Deserialize)] +pub(super) struct JobListParams { + target_id: Option, + limit: Option, + offset: Option, +} + +pub(super) async fn jobs( + State(state): State>, + Extension(owner): Extension, + Query(params): Query, +) -> Result>, AppError> { + if let Some(target) = params.target_id { + check_target_owner(&state.repo, target, &owner).await?; + } + Ok(Json( + state + .repo + .list_workbench_jobs( + owner.0, + params.target_id, + clamp_limit(params.limit), + clamp_offset(params.offset), + ) + .await + .map_err(AppError::internal)?, + )) +} + +#[derive(Deserialize)] +pub(super) struct ImportRequest { + records: Vec, +} + +#[derive(Deserialize)] +struct ImportRecord { + tx_hash: String, + timestamp: i64, + #[serde(default)] + network: Option, + #[serde(default)] + block_number: Option, + raw_metadata: serde_json::Value, +} + +fn validate_records(records: &[ImportRecord], target: &IndexTarget) -> Result<(), AppError> { + if records.is_empty() || records.len() > 500 { + return Err(AppError::bad_request( + "Import requires 1–500 records (maximum request size 1 MiB)", + )); + } + let mut hashes = HashSet::new(); + for (index, record) in records.iter().enumerate() { + let invalid = + |message: &str| AppError::bad_request(format!("Record {}: {message}", index + 1)); + if record.tx_hash.is_empty() + || record.tx_hash.len() > 256 + || record.tx_hash.chars().any(char::is_whitespace) + { + return Err(invalid( + "tx_hash must contain 1–256 non-whitespace characters", + )); + } + if !hashes.insert(&record.tx_hash) { + return Err(invalid("duplicate tx_hash in import")); + } + if record.timestamp < 0 + || record.timestamp > 32_503_680_000 + || record.block_number.is_some_and(|b| b < 0) + { + return Err(invalid("invalid timestamp (Unix seconds) or block_number")); + } + if record + .network + .as_ref() + .is_some_and(|n| n != &target.network) + { + return Err(invalid("network does not match target")); + } + spectraplex_adapters::validate_import_payload(target.chain_family, &record.raw_metadata) + .map_err(|e| invalid(&e.to_string()))?; + } + Ok(()) +} + +async fn import( + state: &AppState, + target: &IndexTarget, + records: Vec, + source: &str, +) -> Result, AppError> { + if let Some(wallet) = target.address.as_deref() { + check_wallet_allowed(wallet, &state.allowed_wallets)?; + } + validate_records(&records, target)?; + let now = chrono::Utc::now(); + let run = IngestionRun { + id: Uuid::new_v4(), + target_id: Some(target.id), + network: target.network.clone(), + source: source.into(), + mode: IngestionJobMode::Backfill, + status: IngestionJobStatus::Completed, + started_at: now, + finished_at: Some(now), + records_written: records.len() as i64, + error_message: None, + cursor_state: Some(serde_json::json!({"coverage": "imported records only"})), + }; + let raw: Vec<_> = records + .into_iter() + .map(|r| RawTransaction { + id: Uuid::new_v4(), + network: target.network.clone(), + tx_hash: r.tx_hash, + timestamp: r.timestamp, + block_number: r.block_number, + raw_metadata: r.raw_metadata, + source: source.into(), + ingestion_run_id: Some(run.id), + ingested_at: now, + }) + .collect(); + let job = state + .repo + .import_raw_records(target, &run, &raw) + .await + .map_err(AppError::internal)?; + Ok(Json( + serde_json::json!({"target_id": target.id, "ingestion_run_id": run.id, + "materialization_job_id": job, "record_count": raw.len()}), + )) +} + +pub(super) async fn import_records( + State(state): State>, + Extension(owner): Extension, + Path(id): Path, + Json(request): Json, +) -> Result, AppError> { + // Unverified payloads must not let a tenant poison canonical chain data for others. + if owner.0.is_some() { + return Err(AppError::forbidden( + "File import requires an admin key; tenants can use sample data or provider ingestion", + )); + } + let target = state + .repo + .get_index_target(id) + .await + .map_err(AppError::internal)? + .ok_or_else(|| AppError::not_found("Target not found"))?; + if target.network.ends_with("-demo") { + return Err(AppError::bad_request( + "Sample networks accept only bundled sample data", + )); + } + import(&state, &target, request.records, "trusted-import").await +} + +pub(super) async fn demo( + State(state): State>, + Extension(owner): Extension, + Path(chain): Path, +) -> Result, AppError> { + #[derive(Deserialize)] + struct Sample { + network: String, + wallet: String, + label: String, + records: Vec, + } + let mut samples: HashMap = + serde_json::from_str(include_str!("../static/samples.json")).map_err(AppError::internal)?; + let sample = samples + .remove(&chain) + .ok_or_else(|| AppError::bad_request("Choose solana, ethereum or hyperliquid"))?; + check_wallet_allowed(&sample.wallet, &state.allowed_wallets)?; + let existing = state + .repo + .get_index_target_by_address(TargetKind::Wallet, &sample.network, &sample.wallet, owner.0) + .await + .map_err(AppError::internal)?; + let target = if let Some(target) = existing { + target + } else { + let now = chrono::Utc::now(); + let target = IndexTarget { + id: Uuid::new_v4(), + kind: TargetKind::Wallet, + network: sample.network.clone(), + chain_family: DatasetRegistry::chain_family_for_network(&sample.network) + .expect("bundled sample family"), + address: Some(sample.wallet), + filter_spec: None, + mode: TargetMode::Backfill, + label: Some(sample.label), + owner_id: owner.0, + created_at: now, + updated_at: now, + }; + match state.repo.create_index_target(&target).await { + Ok(target) => target, + Err(e) => state + .repo + .get_index_target_by_address( + TargetKind::Wallet, + &target.network, + target.address.as_deref().unwrap(), + owner.0, + ) + .await + .map_err(AppError::internal)? + .ok_or_else(|| AppError::internal(e))?, + } + }; + import(&state, &target, sample.records, "sample-data").await +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn bundled_samples_are_valid() { + let samples: serde_json::Value = + serde_json::from_str(include_str!("../static/samples.json")).unwrap(); + for sample in samples.as_object().unwrap().values() { + let network = sample["network"].as_str().unwrap(); + let now = chrono::Utc::now(); + let target = IndexTarget { + id: Uuid::new_v4(), + kind: TargetKind::Wallet, + network: network.into(), + chain_family: DatasetRegistry::chain_family_for_network(network).unwrap(), + address: Some(sample["wallet"].as_str().unwrap().into()), + filter_spec: None, + mode: TargetMode::Backfill, + label: None, + owner_id: None, + created_at: now, + updated_at: now, + }; + let records: Vec = + serde_json::from_value(sample["records"].clone()).unwrap(); + validate_records(&records, &target).unwrap(); + assert!(validate_records(&[], &target).is_err()); + } + } +} diff --git a/api/src/worker.rs b/api/src/worker.rs index 086cc43..8278f8e 100644 --- a/api/src/worker.rs +++ b/api/src/worker.rs @@ -148,13 +148,30 @@ async fn execute_job( error_message: None, cursor_state: None, }; - let run_created = repo.create_ingestion_run(&run).await.is_ok(); - if !run_created { - warn!(job_id = %job_id, "Failed to create ingestion run; raw_transactions will not carry ingestion_run_id"); + if let Err(error) = repo.create_ingestion_run(&run).await { + heartbeat_cancel.cancel(); + let _ = repo + .fail_ingestion_job( + job_id, + worker_id, + &format!("Cannot create provenance run: {error}"), + ) + .await; + return; } + let run_created = true; // Execute the actual ingestion. - let result = run_ingestion(repo, config, provider_registry, job, run_id, run_created).await; + let result = run_ingestion( + repo, + config, + provider_registry, + job, + run_id, + run_created, + worker_id, + ) + .await; // Stop heartbeat. heartbeat_cancel.cancel(); @@ -205,6 +222,7 @@ async fn execute_job( Ok(Some(target)) => { if let Some(mat_scope) = auto_materialization_scope(&target, &job.network, run_id) + .filter(|_| !target_uses_connector_backfill(&target)) { if let Err(e) = repo .create_materialization_run( @@ -220,13 +238,15 @@ async fn execute_job( } else { info!(job_id = %job_id, "Enqueued post-ingest materialization run"); } - } else if target.kind == TargetKind::Wallet { + } else if target.kind == TargetKind::Wallet + && !target_uses_connector_backfill(&target) + { warn!( job_id = %job_id, target_id = %tid, "Skipping post-ingest materialization for wallet target without an address" ); - } else { + } else if !target_uses_connector_backfill(&target) { info!(job_id = %job_id, target_kind = ?target.kind, "Skipping wallet-scoped post-ingest materialization for non-wallet target"); } } @@ -253,8 +273,8 @@ async fn execute_job( } } Err(e) => { - let err_msg = e.to_string(); - error!(job_id = %job_id, error = %err_msg, "Ingestion job failed"); + let err_msg = public_ingestion_error(&e.to_string()); + error!(job_id = %job_id, error = %e, "Ingestion job failed"); // Update ingestion run. let _ = repo .update_ingestion_run_status( @@ -284,14 +304,33 @@ async fn execute_job( } } +// Upstream errors may echo credential-bearing URLs or provider tokens. Keep +// detailed diagnostics in operator logs, not tenant-visible jobs or callbacks. +fn public_ingestion_error(message: &str) -> String { + if message.starts_with("RPC chain ID mismatch:") + || message.starts_with("More new Solana activity than ingest_limit") + { + message.to_string() + } else { + "Could not ingest activity. Check the server logs and provider configuration, then retry." + .into() + } +} + /// Returns true when a target-centric ingestion job should use the V2 /// Connector abstraction instead of the legacy wallet-shaped ChainIngestor flow. fn target_uses_connector_backfill(target: &IndexTarget) -> bool { matches!( (target.chain_family, target.kind), ( + spectraplex_core::v2::ChainFamily::Solana, + TargetKind::Wallet + ) | ( spectraplex_core::v2::ChainFamily::Hyperliquid, - TargetKind::Market + TargetKind::Wallet | TargetKind::Market + ) | ( + spectraplex_core::v2::ChainFamily::Evm, + TargetKind::Wallet | TargetKind::Contract | TargetKind::TopicFilter ) ) } @@ -303,7 +342,7 @@ async fn run_connector_backfill( target: &IndexTarget, limit: usize, run_id: Uuid, - run_created: bool, + worker_id: &str, ) -> anyhow::Result { let net_ctx = NetworkContext::from_registry(provider_registry, &NetworkId::new(job.network.clone())) @@ -326,7 +365,25 @@ async fn run_connector_backfill( .map(|cp| cp.cursor); let batch: IngestionBatch = match (target.chain_family, target.kind) { - (spectraplex_core::v2::ChainFamily::Hyperliquid, TargetKind::Market) => { + (spectraplex_core::v2::ChainFamily::Solana, TargetKind::Wallet) => { + let mut cursor = checkpoint_cursor + .clone() + .unwrap_or_else(|| serde_json::json!({})); + cursor["incremental"] = + serde_json::json!(job.mode == spectraplex_core::v2::IngestionJobMode::Incremental); + SolanaAdapter::from_network_context(&net_ctx)? + .backfill(target, Some(&cursor), limit) + .await? + } + (spectraplex_core::v2::ChainFamily::Evm, _) => { + EvmAdapter::from_network_context(&net_ctx)? + .backfill(target, checkpoint_cursor.as_ref(), limit) + .await? + } + ( + spectraplex_core::v2::ChainFamily::Hyperliquid, + TargetKind::Wallet | TargetKind::Market, + ) => { let adapter = HyperliquidAdapter::from_network_context(&net_ctx); adapter .backfill(target, checkpoint_cursor.as_ref(), limit) @@ -341,10 +398,8 @@ async fn run_connector_backfill( let count = batch.records.len(); let mut v2_records = batch.records; - if run_created { - for raw in &mut v2_records { - raw.ingestion_run_id = Some(run_id); - } + for raw in &mut v2_records { + raw.ingestion_run_id = Some(run_id); } let mut seen = HashSet::new(); @@ -353,12 +408,6 @@ async fn run_connector_backfill( .filter(|r| seen.insert((r.network.clone(), r.tx_hash.clone()))) .collect(); - let canonical_ids = repo - .upsert_raw_transactions_returning_ids(&v2_deduped) - .await?; - let matches = build_target_matches(target.id, &canonical_ids); - repo.save_target_matches(&matches).await?; - let derived_checkpoint = v2_deduped .iter() .filter_map(|raw| { @@ -378,9 +427,15 @@ async fn run_connector_backfill( updated_at: chrono::Utc::now(), }); - if let Some(cp) = batch.checkpoint.or(derived_checkpoint) { - repo.upsert_checkpoint_v2(&cp).await?; - } + let checkpoint = batch.checkpoint.or(derived_checkpoint); + repo.commit_connector_batch( + target, + run_id, + &v2_deduped, + checkpoint.as_ref(), + (job.id, worker_id), + ) + .await?; Ok(count) } @@ -396,6 +451,7 @@ async fn run_ingestion( job: &IngestionJob, run_id: Uuid, run_created: bool, + worker_id: &str, ) -> anyhow::Result { let target_id = job .target_id @@ -422,7 +478,7 @@ async fn run_ingestion( &target, config.ingest_limit, run_id, - run_created, + worker_id, ) .await; } @@ -787,6 +843,18 @@ mod tests { ); } + #[test] + fn provider_credentials_never_enter_public_job_errors() { + for error in [ + "error sending request for url (https://rpc.example/PRIVATE_KEY)", + "provider: invalid API key PRIVATE_KEY", + ] { + assert!(!public_ingestion_error(error).contains("PRIVATE_KEY")); + } + let mismatch = "RPC chain ID mismatch: expected 1 for ethereum-mainnet, got 2"; + assert_eq!(public_ingestion_error(mismatch), mismatch); + } + #[test] fn target_uses_connector_backfill_for_hyperliquid_market() { let now = chrono::Utc::now(); @@ -808,7 +876,7 @@ mod tests { } #[test] - fn target_uses_connector_backfill_stays_legacy_for_wallets() { + fn target_uses_connector_backfill_for_wallets() { let now = chrono::Utc::now(); let target = IndexTarget { id: Uuid::new_v4(), @@ -824,6 +892,6 @@ mod tests { updated_at: now, }; - assert!(!target_uses_connector_backfill(&target)); + assert!(target_uses_connector_backfill(&target)); } } diff --git a/api/static/app.js b/api/static/app.js new file mode 100644 index 0000000..8c15423 --- /dev/null +++ b/api/static/app.js @@ -0,0 +1,328 @@ +'use strict'; +const $ = id => document.getElementById(id); +const state = { key: '', session: null, targets: [], networks: [], datasets: [], target: null, offset: 0, jobOffset: 0, query: 0, jobs: [], pollBusy: false }; +const PAGE_SIZE = 50; +const busy = new WeakSet(); +let requests = new AbortController(); +const node = (tag, text, className) => { + const el = document.createElement(tag); + if (text !== undefined) el.textContent = text; + if (className) el.className = className; + return el; +}; +function notify(message, error = false) { + $('notice').textContent = message; + $('notice').className = `notice${error ? ' error' : ''}`; + $('notice').setAttribute('role', error ? 'alert' : 'status'); + $('notice').hidden = false; +} +async function api(path, options = {}) { + const response = await fetch(path, { ...options, signal: requests.signal, headers: { Authorization: `Bearer ${state.key}`, ...(options.body ? { 'Content-Type': 'application/json' } : {}) }, cache: 'no-store' }); + if (!response.ok) { + const text = await response.text(); + let message; + try { message = JSON.parse(text).error; } catch { message = text; } + if (response.status === 401 && state.session) disconnect(); + throw new Error(message || `Request failed (${response.status})`); + } + return options.download ? response.blob() : response.status === 204 ? null : response.json(); +} +const post = (path, body) => api(path, { method: 'POST', body: JSON.stringify(body) }); +function run(element, action) { + return async event => { + event?.preventDefault(); + const button = element.tagName === 'FORM' ? element.querySelector('button[type="submit"], button:not([type])') : element; + if (button) { button.disabled = true; busy.add(button); } + try { await action(event); } + catch (error) { if (error.name !== 'AbortError') notify(error.message || 'Cannot reach the API. Check that the server is running.', true); } + finally { if (button) { busy.delete(button); button.disabled = false; } syncActions(); } + }; +} +function disconnect() { + requests.abort(); requests = new AbortController(); + state.key = ''; state.session = null; state.target = null; state.targets = []; state.jobs = []; state.query++; + $('workspace').hidden = true; $('login').hidden = false; $('disconnect').hidden = true; + $('api-key').value = ''; $('new-key').value = ''; $('new-key-box').hidden = true; + $('connection-state').textContent = 'Self-hosted data workbench'; + for (const id of ['target-list', 'keys-table', 'jobs-table', 'records-table']) { + const el = $(id); const body = el.querySelector('tbody'); if (body) body.replaceChildren(); else el.replaceChildren(); + } + $('api-key').focus(); +} +function panel(name) { + for (const button of document.querySelectorAll('[data-panel]')) { + const selected = button.dataset.panel === name; + if (selected) button.setAttribute('aria-current', 'page'); else button.removeAttribute('aria-current'); + $(`panel-${button.dataset.panel}`).hidden = !selected; + } +} +function targetOptions() { + const search = $('target-search').value.toLowerCase(); + $('target-list').replaceChildren(); + for (const target of state.targets.filter(t => `${t.label || ''} ${t.address || ''} ${t.network}`.toLowerCase().includes(search))) { + const option = node('option', target.label || `${target.network} · ${(target.address || '').slice(0, 14)}…`); + option.value = target.id; option.selected = target.id === state.target?.id; + option.title = `${target.network} · ${target.address || ''}`; + $('target-list').append(option); + } + if (!state.target) $('target-list').selectedIndex = -1; +} +async function loadTargets(selectId) { + // Follow the API's bounded pages rather than silently hiding targets beyond the first page. + const targets = []; + for (let offset = 0; ; offset += 1000) { + const page = await api(`/v1/targets?limit=1000&offset=${offset}`); + targets.push(...page); if (page.length < 1000) break; + } + state.targets = targets; + const selected = selectId || state.target?.id || targets[0]?.id; + targetOptions(); + await selectTarget(selected || null); +} +function networkOptions() { + const previous = $('target-network').value; + $('target-network').replaceChildren(); + const market = $('target-kind').value === 'market'; + for (const network of state.networks.filter(n => !n.id.endsWith('-demo') && (!market || n.chain_family === 'hyperliquid'))) { + const configured = state.session.configured_networks.includes(network.id); + const option = node('option', `${network.display_name}${configured ? '' : ' (configure provider / import)'}`); + option.value = network.id; $('target-network').append(option); + } + if ([...$('target-network').options].some(o => o.value === previous)) $('target-network').value = previous; + startBlockField(); +} +function startBlockField() { + $('start-block-field').hidden = state.networks.find(n => n.id === $('target-network').value)?.chain_family !== 'evm'; +} +function syncActions() { + const target = state.target; + $('ingest').disabled = busy.has($('ingest')) || !target || !state.session?.configured_networks.includes(target.network) || target.network.endsWith('-demo'); + for (const [id, disabled] of [['previous', state.offset === 0], ['next', state.recordsCount !== PAGE_SIZE], ['jobs-previous', state.jobOffset === 0], ['jobs-next', state.jobsCount !== PAGE_SIZE]]) { + $(id).disabled = busy.has($(id)) || disabled; + } + $('backfill').hidden = !target || target.chain_family !== 'solana' || target.network.endsWith('-demo'); + $('backfill').disabled = busy.has($('backfill')) || $('ingest').disabled; + $('import-section').hidden = !state.session?.admin || !target || target.network.endsWith('-demo'); +} +async function selectTarget(id) { + state.target = state.targets.find(t => t.id === id) || null; + state.offset = 0; + targetOptions(); syncActions(); + $('empty-target').hidden = !!state.target; $('target-view').hidden = !state.target; + if (!state.target) return; + const target = state.target; + $('selected-target-name').textContent = target.label || 'Untitled target'; + $('selected-target-address').textContent = target.address || target.id; + $('target-network-label').textContent = target.network; + const sample = target.network.endsWith('-demo'); + $('target-caveat').textContent = sample ? 'SYNTHETIC SAMPLE · Not live chain data. Reloading this sample is safe; records are deduplicated.' : target.kind === 'market' ? 'Market targets capture raw funding-rate history. Select Raw transactions to inspect or export it.' : target.chain_family === 'evm' ? 'EVM scans forward in bounded block windows, starting at your chosen block or recent blocks—not full wallet history. The wallet ledger excludes internal transfers and fees beyond execution gas.' : 'Fetch is bounded by your configured ingest limit. It is not a guarantee of full wallet history.'; + const previous = $('dataset').value; + $('dataset').replaceChildren(); + for (const ds of state.datasets.filter(d => d.queryable && d.chain_families.includes(target.chain_family) && (target.kind === 'wallet' || d.tier === 'bronze'))) { + const option = node('option', `${ds.name.replaceAll('_', ' ')} · ${ds.tier}`); option.value = ds.name; $('dataset').append(option); + } + const names = [...$('dataset').options].map(o => o.value); + $('dataset').value = names.includes(previous) ? previous : target.kind === 'wallet' ? 'wallet_ledger' : 'raw_transactions'; + await loadRecords(); +} +function filters() { + const params = new URLSearchParams({ target_id: state.target.id, network: state.target.network }); + const from = $('from').value ? Math.floor(new Date($('from').value).getTime() / 1000) : null; + const to = $('to').value ? Math.floor(new Date($('to').value).getTime() / 1000) : null; + if ((from !== null && !Number.isFinite(from)) || (to !== null && !Number.isFinite(to))) throw new Error('Enter valid dates.'); + if (from !== null && to !== null && from > to) throw new Error('From must be earlier than To.'); + if (from !== null) params.set('time_start', from); + if (to !== null) params.set('time_end', to); + return params; +} +const columns = { + raw_transactions: ['timestamp', 'tx_hash', 'block_number', 'source'], + wallet_ledger: ['timestamp', 'entry_type', 'asset_symbol', 'amount', 'counterparty_address', 'tx_hash'], + balance_history: ['timestamp', 'asset_symbol', 'balance', 'delta'], + token_transfers: ['token_symbol', 'from_address', 'to_address', 'amount', 'decimals'], + native_balance_deltas: ['account_address', 'native_token', 'pre_balance', 'post_balance', 'delta'], + decoded_events: ['event_name', 'program_or_contract', 'event_index'], + hl_fills: ['coin', 'side', 'price', 'size', 'closed_pnl', 'fee'], + hl_funding: ['coin', 'usdc', 'funding_rate'], + hl_pnl_summary: ['coin', 'total_closed_pnl', 'total_funding', 'total_fees', 'net_pnl'], + hl_trade_history: ['coin', 'side', 'entry_price', 'exit_price', 'size', 'realized_pnl'], +}; +function valueText(value, key) { + if (value === null || value === undefined) return '—'; + if (key === 'timestamp' && typeof value === 'number') return new Date(value * 1000).toLocaleString(); + if (typeof value === 'object') return JSON.stringify(value); + return String(value); +} +function renderRecords(records, dataset) { + const head = $('records-table').querySelector('thead'); const body = $('records-table').querySelector('tbody'); + head.replaceChildren(); body.replaceChildren(); + const preferred = columns[dataset] || []; + const fields = records[0] ? (preferred.filter(c => c in records[0]).length ? preferred.filter(c => c in records[0]) : Object.keys(records[0]).filter(c => !['id', 'dataset_version_id', 'raw_transaction_id', 'created_at'].includes(c)).slice(0, 6)) : preferred; + const row = node('tr'); for (const field of [...fields, 'Details']) row.append(node('th', field.replaceAll('_', ' '))); head.append(row); + for (const record of records) { + const tr = node('tr'); + for (const field of fields) { + const value = valueText(record[field], field); + const cell = node('td', value, /amount|balance|price|size|pnl|fee|delta/.test(field) ? 'numeric' : ''); + const identifier = /tx_hash|address|program_or_contract/.test(field) && value.length > 28; + if (identifier) cell.classList.add('identifier'); + if (identifier || value.length > 90) { cell.textContent = `${value.slice(0, identifier ? 12 : 36)}…${value.slice(identifier ? -8 : -12)}`; cell.title = value; } + tr.append(cell); + } + const cell = node('td'); const details = node('details'); details.append(node('summary', 'View JSON'), node('pre', JSON.stringify(record, null, 2))); cell.append(details); tr.append(cell); body.append(tr); + } + $('empty-records').hidden = records.length !== 0; +} +async function loadRecords() { + if (!state.target || !state.key) return; + const query = ++state.query; const dataset = $('dataset').value; const targetId = state.target.id; + const params = filters(); params.set('limit', PAGE_SIZE); params.set('offset', state.offset); + $('record-count').textContent = 'Loading…'; + renderRecords([], dataset); $('empty-records').hidden = true; + let records; + try { records = await api(`/v1/datasets/${dataset}/records?${params}`); } + catch (error) { if (query === state.query) $('record-count').textContent = 'Records unavailable'; throw error; } + if (query !== state.query) return; + state.recordsCount = records.length; + renderRecords(records, dataset); + $('record-count').textContent = `${records.length} records on this page`; + $('previous').disabled = state.offset === 0; $('next').disabled = records.length < PAGE_SIZE; + $('page-label').textContent = `Page ${state.offset / PAGE_SIZE + 1}`; + const completeness = await api(`/v1/datasets/${dataset}/completeness`); + if (query !== state.query) return; + const coverage = completeness.find(c => c.target_id === targetId); + const status = coverage?.status || 'unknown'; + $('coverage').textContent = status === 'complete' ? 'Indexed batch processed' : status === 'unknown' ? 'Coverage unknown' : `Coverage: ${status}`; + $('coverage').className = `badge ${['complete', 'partial', 'gap', 'backfilling'].includes(status) ? status : ''}`; + $('coverage').title = coverage?.notes || 'Coverage refers to indexed data, not proof of complete chain history.'; +} +async function download(job, provenance = false) { + const blob = await api(`/v1/export/jobs/${job.id}/download${provenance ? '?provenance=true' : ''}`, { download: true }); + const url = URL.createObjectURL(blob); const link = node('a'); + link.href = url; link.download = `${job.dataset}-${job.id}.${provenance ? 'provenance.json' : job.format}`; + document.body.append(link); link.click(); link.remove(); setTimeout(() => URL.revokeObjectURL(url), 30000); +} +async function loadJobs() { + if (!state.key) return; + const jobs = await api(`/v1/jobs?limit=${PAGE_SIZE}&offset=${state.jobOffset}`); + const previous = state.jobs; + state.jobs = jobs; + state.jobsCount = jobs.length; + const active = jobs.filter(j => ['pending', 'running', 'claimed', 'delivering'].includes(j.state)); + $('pending-count').textContent = active.length || ''; + const body = $('jobs-table').querySelector('tbody'); body.replaceChildren(); + for (const job of jobs) { + const tr = node('tr'); const operation = node('td', job.kind); operation.title = job.id; tr.append(operation); + tr.append(node('td', job.kind === 'export' ? `${job.dataset} · ${job.format}` : job.network || job.dataset)); + const status = node('td'); status.append(node('span', job.state, `badge ${job.state}`)); tr.append(status); + tr.append(node('td', new Date(job.created_at).toLocaleString())); + const result = node('td'); + if (job.kind === 'export' && job.state === 'completed') { + for (const [label, provenance] of [['Download', false], ['Provenance', true]]) { + const button = node('button', label); button.addEventListener('click', run(button, () => download(job, provenance))); result.append(button); + } + result.append(node('div', `${job.record_count ?? 0} records`, 'hint')); + } else result.textContent = job.message || (job.state === 'completed' ? `${job.record_count ?? ''}${job.record_count !== null ? ' records' : ''}` : ''); + if (job.state === 'failed' && ['materialize', 'ingest'].includes(job.kind)) { + const retry = node('button', 'Retry'); + retry.addEventListener('click', run(retry, async () => { + if (job.kind === 'materialize') await post('/v1/normalize', { wallet: job.wallet, ingestion_run_id: job.ingestion_run_id }); + else await post(`/v1/targets/${job.target_id}/ingest`, { mode: job.mode || 'incremental' }); + notify('Retry queued.'); await loadJobs(); + })); + result.append(node('br'), retry); + } + tr.append(result); body.append(tr); + } + $('empty-jobs').hidden = jobs.length !== 0; + $('jobs-previous').disabled = state.jobOffset === 0; $('jobs-next').disabled = jobs.length < PAGE_SIZE; + $('jobs-page').textContent = `Page ${state.jobOffset / PAGE_SIZE + 1}`; + const justFinished = jobs.some(j => j.kind === 'materialize' && j.state === 'completed' && !previous.some(p => p.id === j.id && p.state === 'completed')); + if (justFinished && state.target && !$('panel-explore').hidden) await loadRecords(); +} +async function loadKeys() { + const keys = await api('/v1/api-keys'); const body = $('keys-table').querySelector('tbody'); body.replaceChildren(); + for (const key of keys) { + const tr = node('tr'); tr.append(node('td', key.name), node('td', key.id.slice(0, 8)), node('td', new Date(key.created_at).toLocaleString())); + const td = node('td'); const button = node('button', 'Revoke'); + button.addEventListener('click', run(button, async () => { + if (!confirm(`Revoke “${key.name}”? Clients using this key will lose access.`)) return; + await api(`/v1/api-keys/${key.id}`, { method: 'DELETE' }); + notify('Key revoked.'); await loadKeys(); + })); td.append(button); tr.append(td); body.append(tr); + } +} +$('login-form').addEventListener('submit', run($('login-form'), async () => { + state.key = $('api-key').value.trim(); + if (!state.key) throw new Error('Enter your API key.'); + state.session = await api('/v1/session'); + [state.networks, state.datasets] = await Promise.all([api('/v1/networks'), api('/v1/datasets')]); + $('api-key').value = ''; $('login').hidden = true; $('workspace').hidden = false; $('disconnect').hidden = false; + $('connection-state').textContent = `${state.session.admin ? 'Admin' : 'Tenant workspace'} · v${state.session.version}`; + $('key-explanation').textContent = state.session.admin ? 'Each new key creates an isolated tenant workspace. Your admin key can access all data; share tenant keys instead.' : 'New keys share your current workspace. Revoking your current key disconnects this session.'; + $('notice').hidden = true; state.offset = 0; state.jobOffset = 0; panel('explore'); networkOptions(); + await loadTargets(); await loadJobs(); +})); +$('disconnect').addEventListener('click', disconnect); +for (const button of document.querySelectorAll('[data-panel]')) button.addEventListener('click', run(button, async () => { panel(button.dataset.panel); if (button.dataset.panel === 'activity') await loadJobs(); if (button.dataset.panel === 'access') await loadKeys(); })); +$('target-search').addEventListener('input', targetOptions); +$('target-list').addEventListener('change', run($('target-list'), () => selectTarget($('target-list').value))); +$('target-kind').addEventListener('change', networkOptions); +$('target-network').addEventListener('change', startBlockField); +$('refresh-targets').addEventListener('click', run($('refresh-targets'), () => loadTargets())); +$('target-form').addEventListener('submit', run($('target-form'), async () => { + const block = $('target-start-block').value; + const filter_spec = !$('start-block-field').hidden && block !== '' ? { from_block: Number(block) } : null; + if (filter_spec && (!Number.isSafeInteger(filter_spec.from_block) || filter_spec.from_block < 0)) throw new Error('Starting block must be a nonnegative integer.'); + const target = await post('/v1/targets', { kind: $('target-kind').value, network: $('target-network').value, address: $('target-address').value.trim(), label: $('target-label').value.trim() || null, mode: 'backfill', filter_spec }); + $('target-address').value = ''; $('target-label').value = ''; $('target-start-block').value = ''; $('add-target').open = false; + notify('Target created. Fetch activity, or import trusted data with an admin key.'); await loadTargets(target.id); +})); +for (const button of document.querySelectorAll('[data-demo]')) button.addEventListener('click', run(button, async () => { + const result = await post(`/v1/demo/${button.dataset.demo}`, {}); + $('from').value = ''; $('to').value = ''; $('dataset').value = 'wallet_ledger'; + notify(`Loaded ${result.record_count} synthetic transactions. Materialization is queued; results appear shortly.`); + await loadTargets(result.target_id); await loadJobs(); +})); +$('ingest').addEventListener('click', run($('ingest'), async () => { + const result = await post(`/v1/targets/${state.target.id}/ingest`, { mode: 'incremental' }); + notify(`Fetch queued (${result.id}). Follow ingestion and materialization in Activity.`); await loadJobs(); +})); +$('backfill').addEventListener('click', run($('backfill'), async () => { + await post(`/v1/targets/${state.target.id}/ingest`, { mode: 'backfill' }); + notify('Older Solana history queued. Repeat to continue backwards through history.'); await loadJobs(); +})); +$('query-form').addEventListener('submit', run($('query-form'), async () => { state.offset = 0; await loadRecords(); })); +$('dataset').addEventListener('change', run($('dataset'), async () => { state.offset = 0; await loadRecords(); })); +$('clear-filters').addEventListener('click', run($('clear-filters'), async () => { $('from').value = ''; $('to').value = ''; state.offset = 0; await loadRecords(); })); +for (const [id, delta] of [['previous', -PAGE_SIZE], ['next', PAGE_SIZE]]) $(id).addEventListener('click', run($(id), async () => { state.offset = Math.max(0, state.offset + delta); await loadRecords(); })); +for (const button of document.querySelectorAll('[data-export]')) button.addEventListener('click', run(button, async () => { + const body = Object.fromEntries(filters()); for (const key of ['time_start', 'time_end']) if (key in body) body[key] = Number(body[key]); + await post('/v1/export/dataset', { ...body, dataset: $('dataset').value, format: button.dataset.export }); + notify('Export queued. Download the artifact and its provenance here when completed.'); state.jobOffset = 0; panel('activity'); await loadJobs(); +})); +$('import-form').addEventListener('submit', run($('import-form'), async () => { + const file = $('import-file').files[0]; if (!file) throw new Error('Choose a JSON or JSONL file.'); + if (file.size > 1048576) throw new Error('File exceeds 1 MiB. Split it into smaller batches.'); + const text = await file.text(); let parsed; + try { parsed = JSON.parse(text); } catch { try { parsed = text.split(/\r?\n/).filter(line => line.trim()).map(line => JSON.parse(line)); } catch { throw new Error('Invalid JSON/JSONL file.'); } } + const records = Array.isArray(parsed) ? parsed : parsed.records; + if (!Array.isArray(records)) throw new Error('Expected an array of records or {"records": […]}.'); + const result = await post(`/v1/targets/${state.target.id}/import`, { records }); + notify(`Imported ${result.record_count} records. ${result.materialization_job_id ? 'Materialization queued.' : 'Raw data is ready.'}`); + $('import-file').value = ''; await loadJobs(); await loadRecords(); +})); +$('refresh-jobs').addEventListener('click', run($('refresh-jobs'), loadJobs)); +for (const [id, delta] of [['jobs-previous', -PAGE_SIZE], ['jobs-next', PAGE_SIZE]]) $(id).addEventListener('click', run($(id), async () => { state.jobOffset = Math.max(0, state.jobOffset + delta); await loadJobs(); })); +$('key-form').addEventListener('submit', run($('key-form'), async () => { + const result = await post('/v1/api-keys', { name: $('key-name').value.trim() }); + $('new-key').value = result.key; $('new-key-box').hidden = false; $('key-name').value = ''; await loadKeys(); +})); +$('copy-key').addEventListener('click', run($('copy-key'), async () => { await navigator.clipboard.writeText($('new-key').value); notify('Key copied. Store it securely.'); })); +setInterval(async () => { + if (!state.session || state.pollBusy || document.hidden) return; + state.pollBusy = true; + try { await loadJobs(); } + catch (error) { if (error.name !== 'AbortError') notify(`Refresh failed: ${error.message}`, true); } + finally { state.pollBusy = false; } +}, 4000); diff --git a/api/static/index.html b/api/static/index.html new file mode 100644 index 0000000..9f9141e --- /dev/null +++ b/api/static/index.html @@ -0,0 +1,53 @@ + + + + + + Spectraplex · Data workbench + + + + + +
spectraplex 1.0
Self-hosted data workbench
+
+
+

YOUR DATA. YOUR INFRASTRUCTURE.

Make on-chain activity
work for you.

From raw transactions to useful datasets. Track wallets across Solana, Ethereum and Hyperliquid, then take your data anywhere.

Wallet researchPortfolio activityCSV & JSONL exports
+
CONNECT TO THIS INSTANCE

Open your workspace

Use the key in your .env file, or a tenant key shared by your administrator. Your key stays in memory—not browser storage.

New here? Run ./scripts/setup.sh, then docker compose up --build -d.

+
+ + +
+
spectraplex Open-source blockchain data workbenchSource ↗
+ + + diff --git a/api/static/samples.json b/api/static/samples.json new file mode 100644 index 0000000..c3f1868 --- /dev/null +++ b/api/static/samples.json @@ -0,0 +1,259 @@ +{ + "solana": { + "network": "solana-demo", + "wallet": "So11111111111111111111111111111111111111112", + "label": "Sample \u00b7 Solana wallet", + "records": [ + { + "tx_hash": "sample-solana-1", + "timestamp": 1735693200, + "block_number": 300000001, + "raw_metadata": { + "slot": 300000001, + "blockTime": 1735693200, + "version": 0, + "transaction": { + "signatures": [ + "5555555555555555555555555555555555555555555555555555555555555555555555555555555555555555" + ], + "message": { + "header": { + "numRequiredSignatures": 1, + "numReadonlySignedAccounts": 0, + "numReadonlyUnsignedAccounts": 0 + }, + "accountKeys": [ + "So11111111111111111111111111111111111111112", + "11111111111111111111111111111111" + ], + "recentBlockhash": "11111111111111111111111111111111", + "instructions": [] + } + }, + "meta": { + "err": null, + "status": { + "Ok": null + }, + "fee": 5000, + "preBalances": [ + 2000000000, + 0 + ], + "postBalances": [ + 1749995000, + 250000000 + ], + "preTokenBalances": [], + "postTokenBalances": [], + "innerInstructions": [], + "logMessages": [], + "loadedAddresses": { + "writable": [], + "readonly": [] + } + } + } + }, + { + "tx_hash": "sample-solana-2", + "timestamp": 1735696800, + "block_number": 300000002, + "raw_metadata": { + "slot": 300000002, + "blockTime": 1735696800, + "version": 0, + "transaction": { + "signatures": [ + "5555555555555555555555555555555555555555555555555555555555555555555555555555555555555555" + ], + "message": { + "header": { + "numRequiredSignatures": 1, + "numReadonlySignedAccounts": 0, + "numReadonlyUnsignedAccounts": 0 + }, + "accountKeys": [ + "11111111111111111111111111111111", + "So11111111111111111111111111111111111111112" + ], + "recentBlockhash": "11111111111111111111111111111111", + "instructions": [] + } + }, + "meta": { + "err": null, + "status": { + "Ok": null + }, + "fee": 5000, + "preBalances": [ + 2000000000, + 1749995000 + ], + "postBalances": [ + 999995000, + 2749995000 + ], + "preTokenBalances": [ + { + "accountIndex": 1, + "mint": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + "owner": "So11111111111111111111111111111111111111112", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "uiTokenAmount": { + "amount": "0", + "decimals": 6, + "uiAmount": 0.0, + "uiAmountString": "0.0" + } + } + ], + "postTokenBalances": [ + { + "accountIndex": 1, + "mint": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + "owner": "So11111111111111111111111111111111111111112", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "uiTokenAmount": { + "amount": "150000000", + "decimals": 6, + "uiAmount": 150.0, + "uiAmountString": "150.0" + } + } + ], + "innerInstructions": [], + "logMessages": [], + "loadedAddresses": { + "writable": [], + "readonly": [] + } + } + } + } + ] + }, + "ethereum": { + "network": "ethereum-demo", + "wallet": "0x1111111111111111111111111111111111111111", + "label": "Sample \u00b7 Ethereum wallet", + "records": [ + { + "tx_hash": "0x0000000000000000000000000000000000000000000000000000000000000001", + "timestamp": 1735693200, + "block_number": 21000001, + "raw_metadata": { + "address": "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", + "topics": [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x0000000000000000000000002222222222222222222222222222222222222222", + "0x0000000000000000000000001111111111111111111111111111111111111111" + ], + "data": "0x000000000000000000000000000000000000000000000000000000000ee6b280", + "logIndex": "0x1", + "transactionHash": "0x0000000000000000000000000000000000000000000000000000000000000001" + } + }, + { + "tx_hash": "0x0000000000000000000000000000000000000000000000000000000000000002", + "timestamp": 1735696800, + "block_number": 21000002, + "raw_metadata": { + "address": "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", + "topics": [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x0000000000000000000000001111111111111111111111111111111111111111", + "0x0000000000000000000000002222222222222222222222222222222222222222" + ], + "data": "0x00000000000000000000000000000000000000000000000000000000047868c0", + "logIndex": "0x2", + "transactionHash": "0x0000000000000000000000000000000000000000000000000000000000000002" + } + } + ] + }, + "hyperliquid": { + "network": "hypercore-demo", + "wallet": "0x1111111111111111111111111111111111111111", + "label": "Sample \u00b7 Hyperliquid trader", + "records": [ + { + "tx_hash": "sample-hl-1", + "timestamp": 1735693200, + "raw_metadata": { + "type": "ledger_update", + "data": { + "time": 1735693200000, + "hash": "sample-deposit", + "delta": { + "type": "deposit", + "usdc": "1000" + } + } + } + }, + { + "tx_hash": "sample-hl-2", + "timestamp": 1735696800, + "raw_metadata": { + "type": "fill", + "data": { + "time": 1735696800000, + "coin": "ETH", + "px": "2000", + "sz": "1", + "side": "B", + "dir": "Open Long", + "startPosition": "0", + "closedPnl": "0", + "fee": "1", + "feeToken": "USDC", + "hash": "sample-open", + "tid": 1, + "oid": 1, + "crossed": true + } + } + }, + { + "tx_hash": "sample-hl-3", + "timestamp": 1735700400, + "raw_metadata": { + "type": "fill", + "data": { + "time": 1735700400000, + "coin": "ETH", + "px": "2100", + "sz": "1", + "side": "A", + "dir": "Close Long", + "startPosition": "1", + "closedPnl": "100", + "fee": "1.05", + "feeToken": "USDC", + "hash": "sample-close", + "tid": 2, + "oid": 2, + "crossed": true + } + } + }, + { + "tx_hash": "sample-hl-4", + "timestamp": 1735704000, + "raw_metadata": { + "type": "funding", + "data": { + "time": 1735704000000, + "coin": "ETH", + "usdc": "2.5", + "szi": "0", + "fundingRate": "0.0001", + "hash": "sample-funding" + } + } + } + ] + } +} diff --git a/api/static/style.css b/api/static/style.css new file mode 100644 index 0000000..932a70a --- /dev/null +++ b/api/static/style.css @@ -0,0 +1 @@ +:root{color-scheme:light;--ink:#17243a;--muted:#5b6b80;--line:#dce3ea;--bg:#f3f6f8;--accent:#10765c;--soft:#e4f4ed;--nav:#101c30;--danger:#ad2940;font:14px/1.55 system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif}*{box-sizing:border-box}body{margin:0;color:var(--ink);background:var(--bg)}[hidden]{display:none!important}button,input,select{font:inherit}button,a,input,select,summary{outline-offset:4px}button:focus-visible,a:focus-visible,input:focus-visible,select:focus-visible,summary:focus-visible{outline:3px solid #008b70}button{cursor:pointer;padding:9px 15px;border:1px solid var(--line);border-radius:7px;background:#fff;color:var(--ink);font-weight:600;white-space:nowrap}button:hover:not(:disabled){background:#eef3f5;border-color:#a3b4c4}button:disabled{cursor:not-allowed;opacity:.45}button.primary{background:var(--accent);border-color:var(--accent);color:white}button.primary:hover:not(:disabled){background:#075e48}.text-button{background:none;border-color:transparent;padding:6px 4px;color:var(--accent)}.ghost{background:transparent;color:white;border-color:#586478}input,select{width:100%;padding:10px 11px;border:1px solid #b9c6d2;border-radius:6px;background:white;color:var(--ink)}input::placeholder{color:#6c7b8a}label{display:block;font-weight:600;font-size:12px;margin:0 0 6px}form>label:not(:first-child){margin-top:15px}h1,h2,h3,p{margin-top:0}h1{font-size:28px;line-height:1.2;letter-spacing:-1px;margin-bottom:10px}h2{font-size:19px;line-height:1.3;letter-spacing:-.4px}h3{font-size:16px}.topbar{background:var(--nav);color:white;display:flex;justify-content:space-between;align-items:center;padding:17px 4vw;gap:20px}.brand{font-size:21px;letter-spacing:-.6px;font-weight:750;text-decoration:none;color:white;display:flex;align-items:center;gap:10px}.mark{font-size:29px;color:#6ae3bb;line-height:1}.version{font-size:10px;background:#26354c;border:1px solid #41516a;border-radius:4px;padding:2px 5px;letter-spacing:0;color:#d9e5ee}.connection{display:flex;align-items:center;gap:18px;font-size:12px;color:#c5d1e1}.connection button{font-size:12px;padding:6px 10px}main{max-width:1530px;margin:auto;padding:0 4vw;min-height:calc(100vh - 145px)}.welcome{display:grid;grid-template-columns:1.3fr 1fr;gap:9vw;align-items:center;padding:90px 0 75px}.welcome h1{font-size:clamp(34px,4vw,57px);letter-spacing:-2px;line-height:1.1;margin:18px 0 24px}.lead{font-size:17px;color:var(--muted);line-height:1.7;max-width:570px}.eyebrow{font-size:10px;font-weight:750;letter-spacing:1.7px;color:var(--accent);margin-bottom:12px}.feature-line{display:flex;flex-wrap:wrap;gap:14px;color:var(--muted);font-size:12px;margin-top:28px}.feature-line span:before{content:'✓';color:var(--accent);padding-right:5px}.card{border:1px solid var(--line);border-radius:10px;background:white;box-shadow:0 2px 3px #14223304}.connect-card{padding:30px;max-width:435px}.connect-card h2{font-size:25px;margin:12px 0 25px}.connect-card .primary{width:100%;margin:8px 0 18px}.hint{font-size:12px;color:var(--muted);font-weight:400}.connect-card .hint{margin:12px 0 0}code{font-family:ui-monospace,SFMono-Regular,monospace;font-size:11px;background:#eef2f5;padding:2px 4px;border-radius:3px;overflow-wrap:anywhere}.tabs{display:flex;align-items:center;gap:24px;border-bottom:1px solid var(--line);margin-bottom:28px;padding-top:16px}.tabs button{border:0;border-bottom:3px solid transparent;background:none;padding:13px 0;border-radius:0;color:var(--muted)}.tabs button[aria-current]{border-bottom-color:var(--accent);color:var(--accent)}.local-note{margin-left:auto;color:var(--muted);font-size:11px}.layout{display:grid;grid-template-columns:248px minmax(0,1fr);gap:32px}.explorer,.sidebar{min-width:0}.sidebar{border-right:1px solid var(--line);padding-right:24px}.section-title{display:flex;align-items:center;justify-content:space-between;margin-bottom:10px}.section-title h2{font-size:14px;margin:0}.section-title button{font-size:11px}#target-search{font-size:12px}#target-list{margin:10px 0;border:0;background:transparent;padding:0;height:228px;white-space:normal}#target-list option{padding:10px 8px;border-radius:5px;font-size:12px;overflow:hidden;text-overflow:ellipsis}#target-list option:checked{background:var(--soft);color:#075e48;font-weight:650}details summary{cursor:pointer;font-weight:650;padding:10px 0}details form{margin:10px 0 20px}details form>button{margin-top:16px}.samples{border-top:1px solid var(--line);padding-top:22px;margin-top:16px}.samples .eyebrow{font-size:9px}.sample-buttons{display:flex;flex-wrap:wrap;gap:6px;margin:12px 0}.sample-buttons button{font-size:11px;padding:7px 9px}.samples a{color:var(--accent)}.target-heading,.page-heading{display:flex;align-items:flex-start;justify-content:space-between;gap:18px;margin:6px 0 12px}.target-heading h1{font-size:25px}.target-heading .eyebrow{margin-bottom:6px}.mono{font:11px/1.5 ui-monospace,SFMono-Regular,monospace;overflow-wrap:anywhere}.muted{color:var(--muted)}.target-heading p:last-child{margin-bottom:0}.target-heading button{margin-top:14px}.filters{display:flex;align-items:end;gap:12px;flex-wrap:wrap;border-bottom:1px solid var(--line);padding:19px}.filters>div{flex:1;min-width:155px}.filters .dataset-field{flex:1.3}.filters input,.filters select{font-size:12px;padding:9px}.filters button{font-size:12px}.filters label .hint{font-size:10px}.data-toolbar{padding:18px 19px;display:flex;gap:12px;justify-content:space-between;align-items:center;font-size:12px}.badge{font-size:10px;border-radius:5px;background:#edf2f7;color:#4b5b70;padding:4px 6px;display:inline-block;margin-left:8px}.badge.complete,.badge.completed{color:#086147;background:#e4f4ed}.badge.partial,.badge.pending,.badge.running,.badge.claimed,.badge.delivering{color:#7b5109;background:#fff3d7}.badge.failed,.badge.gap{color:var(--danger);background:#ffebef}.actions{display:flex;gap:7px}.actions button{font-size:11px;padding:7px 10px}.table-scroll{overflow:auto;max-width:100%}table{border-collapse:collapse;min-width:100%;font-size:12px;text-align:left}th{font-weight:600;color:var(--muted);background:#f8fafb;font-size:10px;text-transform:uppercase;letter-spacing:.6px;white-space:nowrap}th,td{padding:12px 17px;border-bottom:1px solid #edf0f4;vertical-align:top}td{max-width:300px;overflow-wrap:anywhere}td.numeric,td.identifier{font-family:ui-monospace,SFMono-Regular,monospace;white-space:nowrap}tbody tr:hover{background:#fafcfc}td details summary{padding:0;color:var(--accent);font-size:11px;font-weight:500}pre{font:11px/1.5 ui-monospace,SFMono-Regular,monospace;white-space:pre-wrap;min-width:250px;max-height:360px;overflow:auto;color:#385066}td button{font-size:11px;padding:5px 9px;margin:0 5px 5px 0}.empty-state{text-align:center;padding:60px 25px;color:var(--muted)}.empty-state h2,.empty-state h3{color:var(--ink)}.empty-state p{font-size:13px}.empty-state.card{margin-top:6px;padding:80px 20px}.empty-mark{display:block;font-size:50px;color:#3c9d81;margin:0 0 22px}.pagination{display:flex;align-items:center;justify-content:space-between;padding:15px 19px;font-size:11px;color:var(--muted)}.pagination button{font-size:11px;padding:6px 10px}.import-card{margin-top:18px;padding:9px 19px}.import-card .hint{max-width:740px}.import-card form{display:flex;align-items:center;gap:16px;flex-wrap:wrap}.import-card input{flex:1;min-width:200px}.import-card form>button{margin:0}.footnote{font-size:11px;color:var(--muted);margin-top:18px;max-width:850px}.page-heading{margin:30px 0 25px}.page-heading p{margin-bottom:0;font-size:13px}.page-heading .eyebrow{font-size:10px;margin-bottom:10px}.key-form{padding:22px;display:flex;gap:16px;align-items:end;margin-bottom:18px}.key-form>div{width:320px}#new-key-box{padding:22px;margin:18px 0}#new-key{font:12px ui-monospace,monospace;max-width:620px;margin:0 8px 12px 0}.notice{border-radius:7px;padding:12px 18px;background:#e4f4ed;border:1px solid #a2cdbd;color:#075e48;font-size:13px;margin:20px 0;overflow-wrap:anywhere}.notice.error{background:#fff0f3;border-color:#eab7c2;color:#9a2138}.count:not(:empty){background:var(--soft);color:var(--accent);border-radius:10px;padding:1px 6px;font-size:10px;margin-left:4px}footer{display:flex;align-items:center;gap:18px;color:var(--muted);padding:26px 4vw;font-size:11px;max-width:1530px;margin:30px auto 0}footer>a{margin-left:auto;color:var(--muted)}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}.skip{position:absolute;left:10px;top:-100px;background:white;padding:12px;z-index:10}.skip:focus{top:10px}@media(min-width:1600px){main{padding:0 40px}}@media(max-width:1000px){.layout{grid-template-columns:215px minmax(0,1fr);gap:22px}.sidebar{padding-right:18px}.local-note{display:none}.data-toolbar{align-items:flex-start;flex-direction:column}.welcome{gap:35px}.connect-card{padding:24px}.filters{gap:10px}}@media(max-width:720px){.topbar{padding:15px 5vw;flex-wrap:wrap}.connection{font-size:10px}.connection>span{max-width:100px}.brand{font-size:18px}.welcome{grid-template-columns:1fr;padding:45px 0;gap:28px}.welcome h1{font-size:39px}.connect-card{max-width:none}.feature-line{margin-top:18px;font-size:11px}.layout{grid-template-columns:1fr}.sidebar{border-right:0;border-bottom:1px solid var(--line);padding:0 0 16px}#target-list{height:112px}#target-search{max-width:none}.samples{padding-top:14px}.sample-buttons{display:inline-flex;margin:0 10px 8px 0}.samples>p{display:inline-block;margin:0 8px 8px 0}.tabs{gap:23px}.tabs button{font-size:12px}.target-heading{flex-wrap:wrap}.target-heading h1{font-size:24px}.target-heading button{margin-top:0}.empty-state.card{padding:45px 15px}.key-form{flex-wrap:wrap}.key-form>div{width:100%}.filters>div{min-width:135px}.filters .dataset-field{flex-basis:100%}footer span{display:none}.pagination{padding:13px}th,td{padding:10px 12px}}@media(prefers-reduced-motion:no-preference){button{transition:background .12s ease,border-color .12s ease}} diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 487aecc..3fc7cc0 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,7 +1,10 @@ [package] name = "spectraplex-cli" -version = "0.1.0" -edition = "2021" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true [dependencies] spectraplex-core = { path = "../core" } diff --git a/cli/src/main.rs b/cli/src/main.rs index a706640..b84a871 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -26,7 +26,7 @@ use tracing::{error, info, warn}; use uuid::Uuid; #[derive(Parser)] -#[command(about = "Spectraplex CLI", long_about = None)] +#[command(version, about = "Spectraplex CLI (direct database / legacy file tools)", long_about = None)] struct Cli { #[arg(global = true, long, env = "DATABASE_URL")] db_url: Option, diff --git a/core/Cargo.toml b/core/Cargo.toml index d7816d6..2e85726 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -1,7 +1,10 @@ [package] name = "spectraplex-core" -version = "0.1.0" -edition = "2021" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true [dependencies] serde = { version = "1.0", features = ["derive"] } diff --git a/core/src/config.rs b/core/src/config.rs index bbe36e6..02cd059 100644 --- a/core/src/config.rs +++ b/core/src/config.rs @@ -63,12 +63,12 @@ impl std::fmt::Debug for ProviderConfig { f.debug_struct("ProviderConfig") .field("network", &self.network) .field("kind", &self.kind) - .field("url", &self.url) + .field("url", &"[REDACTED]") .field("priority", &self.priority) .field("capabilities", &self.capabilities) .field("token_env", &"[REDACTED]") .field("token", &"[REDACTED]") - .field("headers", &self.headers) + .field("headers", &"[REDACTED]") .finish() } } @@ -246,6 +246,20 @@ impl AppConfig { "export_dir must not be empty".to_string(), )); } + if self + .api_key + .as_ref() + .is_some_and(|key| key.trim().is_empty()) + { + return Err(ConfigError::Validation( + "api_key must not be blank".to_string(), + )); + } + if self.ingest_limit == 0 || self.ingest_limit > 100_000 { + return Err(ConfigError::Validation( + "ingest_limit must be between 1 and 100000".to_string(), + )); + } if let Some(ref secret) = self.callback_hmac_secret { if secret.trim().is_empty() { return Err(ConfigError::Validation( @@ -257,9 +271,24 @@ impl AppConfig { } pub fn load() -> Result> { + let path = std::env::var_os("SPECTRAPLEX_CONFIG"); + if let Some(ref path) = path { + if !std::path::Path::new(path).is_file() { + return Err(Box::new(figment::Error::from( + "SPECTRAPLEX_CONFIG must name an existing file", + ))); + } + } + Self::load_from( + path.as_deref() + .unwrap_or_else(|| std::ffi::OsStr::new("spectraplex.toml")), + ) + } + + pub fn load_from(path: impl AsRef) -> Result> { Figment::new() .merge(Serialized::defaults(AppConfig::default())) - .merge(Toml::file("spectraplex.toml")) + .merge(Toml::file(path)) .merge(Env::prefixed("SPECTRAPLEX_")) .merge(Env::raw().only(&[ "DATABASE_URL", @@ -487,6 +516,31 @@ mod tests { } } + #[test] + fn test_load_explicit_config_file() { + let path = + std::env::temp_dir().join(format!("spectraplex-config-{}.toml", uuid::Uuid::new_v4())); + std::fs::write(&path, "ingest_limit = 4321\n").unwrap(); + let config = AppConfig::load_from(&path).unwrap(); + std::fs::remove_file(path).unwrap(); + assert_eq!(config.ingest_limit, 4321); + } + + #[test] + fn test_invalid_ingest_limits_and_blank_key() { + let mut config = AppConfig { + database_url: "postgres://localhost/test".into(), + ..AppConfig::default() + }; + for limit in [0, 100_001] { + config.ingest_limit = limit; + assert!(config.validate().is_err()); + } + config.ingest_limit = 50; + config.api_key = Some(" ".into()); + assert!(config.validate().is_err()); + } + #[test] fn test_allowed_wallets_set_none() { let config = AppConfig::default(); diff --git a/core/src/connector.rs b/core/src/connector.rs index 8941d73..05022a5 100644 --- a/core/src/connector.rs +++ b/core/src/connector.rs @@ -348,6 +348,18 @@ pub fn validate_target(target: &IndexTarget) -> Result<(), Vec> { )); } + if let Some(block) = target + .filter_spec + .as_ref() + .and_then(|s| s.get("from_block")) + { + if target.chain_family != ChainFamily::Evm + || block.as_u64().is_none_or(|n| n > i64::MAX as u64) + { + errors.push("from_block must be a nonnegative integer up to 9223372036854775807 on an EVM target".into()); + } + } + // Rule 3: filter_spec required for complex target kinds let filter_required = matches!(target.kind, TargetKind::TopicFilter | TargetKind::Protocol); if filter_required && target.filter_spec.is_none() { @@ -398,6 +410,33 @@ mod tests { } } + #[test] + fn evm_start_block_is_validated() { + for value in [ + serde_json::json!(-1), + serde_json::json!(1.5), + serde_json::json!("1"), + serde_json::json!(u64::MAX), + ] { + let target = make_target( + TargetKind::Wallet, + ChainFamily::Evm, + Some("0x123"), + Some(serde_json::json!({"from_block": value})), + TargetMode::Backfill, + ); + assert!(validate_target(&target).is_err()); + } + let target = make_target( + TargetKind::Wallet, + ChainFamily::Evm, + Some("0x123"), + Some(serde_json::json!({"from_block": 0})), + TargetMode::Backfill, + ); + assert!(validate_target(&target).is_ok()); + } + // -- ConnectorCapabilities -- #[test] diff --git a/core/src/materializer.rs b/core/src/materializer.rs index c2ddcfe..7ee2e42 100644 --- a/core/src/materializer.rs +++ b/core/src/materializer.rs @@ -28,6 +28,8 @@ use crate::v2::ChainFamily; #[serde(rename_all = "snake_case")] #[strum(serialize_all = "snake_case")] pub enum DatasetName { + /// Canonical Bronze payloads, including non-wallet target data. + RawTransactions, /// Financial ledger for tax/portfolio (existing Silver dataset). LedgerEntries, /// Canonical token transfer records across chains. @@ -64,6 +66,7 @@ impl DatasetName { /// serialization of the enum. pub fn as_sql_str(&self) -> &'static str { match self { + DatasetName::RawTransactions => "raw_transactions", DatasetName::LedgerEntries => "ledger_entries", DatasetName::TokenTransfers => "token_transfers", DatasetName::NativeBalanceDeltas => "native_balance_deltas", @@ -92,6 +95,7 @@ impl DatasetName { /// | `positions` | `hl_position_changes` | pub fn physical_table(&self) -> &'static str { match self { + DatasetName::RawTransactions => "raw_transactions", DatasetName::LedgerEntries => "ledger_entries", DatasetName::TokenTransfers => "token_transfers", DatasetName::NativeBalanceDeltas => "native_balance_deltas", @@ -127,6 +131,7 @@ impl DatasetName { /// Returns the dataset tier (Bronze, Silver, or Gold). pub fn tier(&self) -> DatasetTier { match self { + DatasetName::RawTransactions => DatasetTier::Bronze, DatasetName::LedgerEntries | DatasetName::TokenTransfers | DatasetName::NativeBalanceDeltas @@ -147,7 +152,7 @@ impl DatasetName { /// Returns the chain families that can produce this dataset. pub fn chain_families(&self) -> &'static [ChainFamily] { match self { - DatasetName::TokenTransfers => &[ + DatasetName::RawTransactions | DatasetName::TokenTransfers => &[ ChainFamily::Solana, ChainFamily::Evm, ChainFamily::Hyperliquid, @@ -178,6 +183,7 @@ impl DatasetName { /// Returns all dataset names as a slice. pub fn all() -> &'static [DatasetName] { &[ + DatasetName::RawTransactions, DatasetName::LedgerEntries, DatasetName::TokenTransfers, DatasetName::NativeBalanceDeltas, @@ -272,6 +278,7 @@ impl DatasetRegistry { /// `/v1/ledger/:wallet` endpoint. pub fn queryable() -> &'static [DatasetName] { &[ + DatasetName::RawTransactions, DatasetName::TokenTransfers, DatasetName::NativeBalanceDeltas, DatasetName::DecodedEvents, @@ -293,6 +300,7 @@ impl DatasetRegistry { /// `/v1/export/:wallet` endpoint. pub fn exportable() -> &'static [DatasetName] { &[ + DatasetName::RawTransactions, DatasetName::TokenTransfers, DatasetName::NativeBalanceDeltas, DatasetName::DecodedEvents, @@ -719,7 +727,7 @@ pub struct TokenTransfer { pub to_address: String, /// Transfer amount (positive = movement from sender to receiver). pub amount: BigDecimal, - /// Token decimals used for normalization. + /// Token decimals used for normalization; -1 means unknown, with amount in raw units. pub decimals: i32, /// Ordinal position within the same (raw_transaction_id, from, to, token) group. pub transfer_index: i32, @@ -1232,14 +1240,15 @@ mod tests { use strum::IntoEnumIterator; #[test] - fn dataset_name_count_is_thirteen() { - assert_eq!(DatasetName::all().len(), 13); - assert_eq!(DatasetName::iter().count(), 13); + fn dataset_name_count_matches_registry() { + assert_eq!(DatasetName::all().len(), 14); + assert_eq!(DatasetName::iter().count(), 14); } #[test] fn dataset_name_serde_roundtrip() { let cases = [ + (DatasetName::RawTransactions, "\"raw_transactions\""), (DatasetName::LedgerEntries, "\"ledger_entries\""), (DatasetName::TokenTransfers, "\"token_transfers\""), ( @@ -1257,7 +1266,11 @@ mod tests { (DatasetName::ProtocolEvents, "\"protocol_events\""), (DatasetName::PoolSnapshots, "\"pool_snapshots\""), ]; - assert_eq!(cases.len(), 13, "must cover all 13 datasets"); + assert_eq!( + cases.len(), + DatasetName::all().len(), + "must cover every dataset" + ); for (variant, expected_json) in cases { let json = serde_json::to_string(&variant).unwrap(); assert_eq!(json, expected_json, "serialize {variant:?}"); @@ -2477,7 +2490,8 @@ mod tests { #[test] fn silver_gold_partitions_cover_all_datasets() { - let mut all_partitioned: Vec = Vec::new(); + let mut all_partitioned = vec![DatasetName::RawTransactions]; + assert_eq!(DatasetName::RawTransactions.tier(), DatasetTier::Bronze); all_partitioned.extend_from_slice(DatasetName::silver()); all_partitioned.extend_from_slice(DatasetName::gold()); all_partitioned.sort_by_key(|d| d.as_sql_str()); @@ -2487,7 +2501,7 @@ mod tests { assert_eq!( all_partitioned, all_datasets, - "silver() + gold() must cover all datasets exactly once" + "Bronze + silver() + gold() must cover all datasets exactly once" ); } @@ -2577,8 +2591,7 @@ mod tests { #[test] fn registry_queryable_contains_all_non_legacy_datasets() { let queryable = DatasetRegistry::queryable(); - // All 12 non-legacy datasets should be queryable. - assert_eq!(queryable.len(), 12); + assert_eq!(queryable.len(), 13); // LedgerEntries should not be in the queryable list. assert!( !queryable.contains(&DatasetName::LedgerEntries), @@ -2589,7 +2602,7 @@ mod tests { #[test] fn registry_exportable_contains_all_non_legacy_datasets() { let exportable = DatasetRegistry::exportable(); - assert_eq!(exportable.len(), 12); + assert_eq!(exportable.len(), 13); assert!( !exportable.contains(&DatasetName::LedgerEntries), "LedgerEntries served via legacy endpoint, not dataset export" diff --git a/docker-compose.yml b/docker-compose.yml index 1782bec..e1d81e4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,33 +1,44 @@ services: postgres: image: postgres:17 + restart: unless-stopped environment: POSTGRES_USER: spectraplex POSTGRES_PASSWORD: spectraplex POSTGRES_DB: spectraplex ports: - - "5432:5432" + - "127.0.0.1:${POSTGRES_PORT:-5432}:5432" volumes: - pgdata:/var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U spectraplex"] interval: 5s timeout: 5s - retries: 5 + retries: 10 api: build: . + user: "${CONTAINER_UID:-1000}:1000" + restart: unless-stopped ports: - - "3000:3000" + - "127.0.0.1:${SPECTRAPLEX_PORT:-3000}:3000" environment: DATABASE_URL: postgres://spectraplex:spectraplex@postgres:5432/spectraplex + SPECTRAPLEX_API_KEY: "${SPECTRAPLEX_API_KEY:?Run ./scripts/setup.sh first}" SPECTRAPLEX_PORT: "3000" SPECTRAPLEX_HOST: "0.0.0.0" SPECTRAPLEX_LOG_LEVEL: info + # Optional environment overrides; otherwise honor spectraplex.toml. + SOLANA_RPC_URL: + EVM_RPC_URL: + volumes: + - exports:/app/exports + - ./spectraplex.toml:/app/spectraplex.toml:ro depends_on: postgres: condition: service_healthy - command: ["spectraplex-api"] + stop_grace_period: 30s volumes: pgdata: + exports: diff --git a/migrations/20260923000000_workbench.sql b/migrations/20260923000000_workbench.sql new file mode 100644 index 0000000..bbad842 --- /dev/null +++ b/migrations/20260923000000_workbench.sql @@ -0,0 +1,20 @@ +-- Incremental is the API's default job mode; it must be accepted by run lineage too. +ALTER TABLE ingestion_runs DROP CONSTRAINT chk_ingestion_runs_mode; +ALTER TABLE ingestion_runs ADD CONSTRAINT chk_ingestion_runs_mode + CHECK (mode IN ('backfill', 'incremental', 'stream', 'both')); + +-- Preserve every run's input set when multiple targets ingest the same transaction. +CREATE TABLE ingestion_run_transactions ( + ingestion_run_id UUID NOT NULL REFERENCES ingestion_runs(id) ON DELETE CASCADE, + raw_transaction_id UUID NOT NULL REFERENCES raw_transactions(id) ON DELETE CASCADE, + PRIMARY KEY (ingestion_run_id, raw_transaction_id) +); +CREATE INDEX idx_run_transactions_raw ON ingestion_run_transactions(raw_transaction_id); +INSERT INTO ingestion_run_transactions +SELECT ingestion_run_id, id FROM raw_transactions WHERE ingestion_run_id IS NOT NULL; + +-- Synthetic examples never share a namespace with real chain data. +INSERT INTO networks (id, chain_family, display_name, is_testnet, finality_model) VALUES + ('solana-demo', 'solana', 'Solana · sample data', TRUE, 'instant'), + ('ethereum-demo', 'evm', 'Ethereum · sample data', TRUE, 'instant'), + ('hypercore-demo', 'hyperliquid', 'Hyperliquid · sample data', TRUE, 'instant'); diff --git a/migrations/20260923010000_balance_event_deltas.sql b/migrations/20260923010000_balance_event_deltas.sql new file mode 100644 index 0000000..3e8392c --- /dev/null +++ b/migrations/20260923010000_balance_event_deltas.sql @@ -0,0 +1,3 @@ +-- Persist event deltas so older arrivals can recompute the existing suffix safely. +-- Legacy rows remain NULL: never invent a delta for an untraceable old snapshot. +ALTER TABLE balance_history ADD COLUMN delta NUMERIC; diff --git a/scripts/container_test.py b/scripts/container_test.py new file mode 100644 index 0000000..bf79547 --- /dev/null +++ b/scripts/container_test.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Check an isolated running deployment before/after container recreation.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import time +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('phase', choices=['prepare', 'verify']) +parser.add_argument('state', type=Path) +args = parser.parse_args() +base = os.environ['SPECTRAPLEX_TEST_URL'].rstrip('/') +assert urlsplit(base).hostname in ['127.0.0.1', 'localhost'], 'Use an isolated local deployment' +key = os.environ['SPECTRAPLEX_TEST_KEY'] + + +def call(path, data=None, raw=False): + request = Request(base + path, data=json.dumps(data).encode() if data is not None else None, + headers={'Authorization': f'Bearer {key}', 'Content-Type': 'application/json'}) + with urlopen(request, timeout=15) as response: + body = response.read() + return body if raw else json.loads(body) + + +def completed(path): + for _ in range(120): + job = call(path) + assert job['state'] != 'failed', job + if job['state'] == 'completed': + return job + time.sleep(.5) + raise AssertionError(f'Timed out: {path}') + + +assert call('/ready')['version'] == '1.0.0' +assert b'Data workbench' in call('/', raw=True) +if args.phase == 'prepare': + sample = call('/v1/demo/ethereum', {}) + completed('/v1/jobs/' + sample['materialization_job_id']) + job = call('/v1/export/dataset', {'dataset': 'raw_transactions', 'target_id': sample['target_id'], 'format': 'jsonl'}) + assert completed('/v1/export/jobs/' + job['id'])['record_count'] == 2 + download = '/v1/export/jobs/' + job['id'] + '/download' + state = {'target': sample['target_id'], 'download': download, + 'digest': hashlib.sha256(call(download, raw=True)).hexdigest()} + args.state.parent.mkdir(parents=True, exist_ok=True) + args.state.touch(mode=0o600, exist_ok=True) + args.state.write_text(json.dumps(state)) + print('PASS container: migrations, embedded UI, pipeline and export volume writes') +else: + state = json.loads(args.state.read_text()) + assert call('/v1/targets/' + state['target'])['id'] == state['target'] + assert len(call('/v1/datasets/wallet_ledger/records?target_id=' + state['target'])) == 2 + assert hashlib.sha256(call(state['download'], raw=True)).hexdigest() == state['digest'] + assert call(state['download'] + '?provenance=true')['record_count'] == 2 + print('PASS container: database, jobs, artifacts and provenance survive recreation') diff --git a/scripts/local-dev.sh b/scripts/local-dev.sh index 3294ac2..d895f1a 100755 --- a/scripts/local-dev.sh +++ b/scripts/local-dev.sh @@ -81,6 +81,7 @@ case "${1:-start}" in ;; esac +"$SCRIPT_DIR/setup.sh" echo "Starting Postgres for local development..." compose up -d postgres @@ -92,7 +93,7 @@ echo "" echo "Postgres is ready on localhost:5432" echo "" echo "Next steps:" -echo " 1. Copy spectraplex.toml.example to spectraplex.toml and edit as needed." -echo " 2. Run: cargo run --bin spectraplex-api" +echo " 1. Run: cargo run --bin spectraplex-api" +echo " 2. Open http://localhost:3000; connect with the key in .env." echo " 3. Test: ./scripts/smoke-test.sh" echo "" diff --git a/scripts/provider_fixtures.py b/scripts/provider_fixtures.py new file mode 100644 index 0000000..bb1ab0e --- /dev/null +++ b/scripts/provider_fixtures.py @@ -0,0 +1,54 @@ +"""Deterministic Ethereum RPC fixture: multi-log transaction and token metadata.""" +from copy import deepcopy + +ZERO = '0x' + '0' * 64 +BLOCK_HASH = '0x' + 'b' * 64 +TX_HASH = '0x' + 'e' * 64 +RECIPIENT = '0x' + '2' * 40 +BLOOM = '0x' + '0' * 512 + + +def evm_response(method, params, wallet, chain_id=1, head=14): + tx = {'hash': TX_HASH, 'blockHash': BLOCK_HASH, 'blockNumber': '0x1', + 'transactionIndex': '0x0', 'from': wallet, 'to': RECIPIENT, 'value': hex(10**18), + 'gas': '0x5208', 'gasPrice': hex(10**9), 'input': '0x', 'nonce': '0x0', + 'type': '0x0', 'v': '0x1b', 'r': '0x1', 's': '0x1'} + logs = [{'address': '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + 'topics': ['0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + '0x' + wallet[2:].rjust(64, '0'), '0x' + RECIPIENT[2:].rjust(64, '0')], + 'data': f'0x{amount * 10**6:064x}', 'logIndex': hex(i), 'removed': False, + 'blockNumber': '0x1', 'blockHash': BLOCK_HASH, 'transactionHash': TX_HASH, + 'transactionIndex': '0x0'} for i, amount in enumerate([10, 20])] + custom = deepcopy(logs[0]) + custom.update(address='0x' + 'a' * 40, data=f'0x{12 * 10**6:064x}', logIndex='0x2') + logs.append(custom) + if method == 'eth_call': + assert params[0] == {'to': custom['address'], 'data': '0x313ce567'} + return '0x' + '6'.rjust(64, '0') + if method == 'eth_chainId': + return hex(chain_id) + if method == 'eth_blockNumber': + return hex(head) + if method == 'eth_getLogs': + spec = params[0] + if int(spec['fromBlock'], 16) <= 1 <= int(spec['toBlock'], 16) and spec['topics'][1]: + return logs + return [] + if method == 'eth_getTransactionByHash': + return tx + if method == 'eth_getTransactionReceipt': + return {'transactionHash': TX_HASH, 'transactionIndex': '0x0', 'blockHash': BLOCK_HASH, + 'blockNumber': '0x1', 'from': wallet, 'to': RECIPIENT, 'cumulativeGasUsed': '0x5208', + 'gasUsed': '0x5208', 'effectiveGasPrice': hex(10**9), 'contractAddress': None, + 'logs': logs, 'logsBloom': BLOOM, 'status': '0x1', 'type': '0x0'} + if method == 'eth_getBlockByNumber': + number = int(params[0], 16) + return {'number': hex(number), 'hash': BLOCK_HASH if number == 1 else ZERO, + 'parentHash': ZERO, 'nonce': '0x0000000000000000', 'sha3Uncles': ZERO, + 'logsBloom': BLOOM, 'transactionsRoot': ZERO, 'stateRoot': ZERO, + 'receiptsRoot': ZERO, 'miner': '0x' + '0' * 40, 'difficulty': '0x0', + 'totalDifficulty': '0x0', 'extraData': '0x', 'size': '0x0', 'gasLimit': '0x1c9c380', + 'gasUsed': '0x5208' if number == 1 else '0x0', 'mixHash': ZERO, + 'timestamp': hex(1735689600 + number * 12), 'uncles': [], + 'transactions': ([deepcopy(tx) if params[1] else TX_HASH] if number == 1 else [])} + raise AssertionError(f'Unexpected EVM RPC method: {method}') diff --git a/scripts/setup.sh b/scripts/setup.sh new file mode 100755 index 0000000..e51a162 --- /dev/null +++ b/scripts/setup.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." +umask 077 +container_uid="$(id -u)" +[[ "$container_uid" != 0 ]] || container_uid=1000 +if [[ ! -f .env ]]; then + command -v openssl >/dev/null || { echo 'openssl is required to generate a secure API key.' >&2; exit 1; } + key="$(openssl rand -hex 32)" + printf 'SPECTRAPLEX_API_KEY=%s\nDATABASE_URL=postgres://spectraplex:spectraplex@localhost:5432/spectraplex\n' "$key" > .env + echo 'Created .env with a random admin key (keep this file private).' +else + echo 'Keeping existing .env.' +fi +if ! grep -q '^CONTAINER_UID=' .env; then + printf '\nCONTAINER_UID=%s\n' "$container_uid" >> .env +fi +if [[ ! -f spectraplex.toml ]]; then + cp spectraplex.toml.example spectraplex.toml + if [[ "$(id -u)" == 0 ]]; then chown "$container_uid" spectraplex.toml; fi +fi +echo 'Run: docker compose up --build -d' +echo 'Then open http://localhost:3000 and connect with SPECTRAPLEX_API_KEY from .env.' diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index a51a487..c1f4781 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -1,397 +1,14 @@ #!/usr/bin/env bash set -euo pipefail - -# --------------------------------------------------------------------------- -# Spectraplex API Smoke Test -# --------------------------------------------------------------------------- -# Proves the supported API path end-to-end on a fresh local environment. -# -# What this verifies: -# - API server boots and responds to health checks -# - Tenant API key creation and authentication work -# - Target registration works -# - Ingestion can be enqueued and its job status polled -# - Tenant-scoped dataset queries accept target_id and return data (if materialized) -# - Tenant-scoped export jobs can be created with a sink config -# - API key and target listing work -# -# What this does NOT verify (these run in background workers): -# - Actual RPC ingestion completion against live providers -# - Silver/Gold materialization (async, may take minutes) -# - Export job completion and file delivery (async worker process) -# -# Steps: -# 1. Start Postgres -# 2. Start the API server -# 3. Create a tenant-scoped API key (using legacy admin key) -# 4. Register a wallet target -# 5. Trigger ingestion (wallet + network) -# 6. Poll job status (terminal state check) -# 7. Query dataset records (tenant-scoped via target_id) -# 8. Create an export job (tenant-scoped via target_id) -# 9. Verify tenant-scoped API key lifecycle -# 10. Clean up -# -# The script fails loudly on any unexpected HTTP response. -# If live providers are unavailable, use --skip-ingest to test the API surface -# without provider-dependent steps. -# --------------------------------------------------------------------------- - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -API_URL="http://localhost:3000" -LEGACY_KEY="admin-smoke-test-key" -TENANT_KEY="" -API_KEY_ID="" -API_PID="" -TARGET_WALLET="So11111111111111111111111111111111111111112" -TARGET_NETWORK="solana-mainnet" - -usage() { - cat <<'USAGE' -Usage: ./scripts/smoke-test.sh [--skip-ingest] - -Options: - --skip-ingest Skip provider-dependent ingestion and job polling. - -h, --help Show this help message. -USAGE -} - -SKIP_INGEST=false -for arg in "$@"; do - case "$arg" in - --skip-ingest) - SKIP_INGEST=true - ;; - -h | --help) - usage - exit 0 - ;; - *) - echo "ERROR: Unknown option: $arg" >&2 - usage >&2 - exit 1 - ;; - esac +cd "$(dirname "${BASH_SOURCE[0]}")/.." +if [[ "${1:-}" == --help || "${1:-}" == -h ]]; then + echo 'Usage: TEST_DATABASE_URL=postgres://... ./scripts/smoke-test.sh [--browser]' + echo 'Creates and removes an isolated database; uses local provider fixtures, not mainnet.' + echo 'Requires Rust, Python 3 and psql. --browser also requires Playwright (see web-tests/package.json).' + exit 0 +fi +for tool in cargo python3 psql; do + command -v "$tool" >/dev/null || { echo "Required command not found: $tool" >&2; exit 1; } done - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -curl_json() { - local method="${1:-GET}" - local url="$2" - local auth="${3:-}" - local body="${4:-}" - - local cmd=(curl -sS -w "\n%{http_code}" -X "$method" "$url") - if [[ -n "$auth" ]]; then - cmd+=(-H "Authorization: Bearer $auth") - fi - cmd+=(-H "Content-Type: application/json") - if [[ -n "$body" ]]; then - cmd+=(-d "$body") - fi - - local resp - if ! resp=$("${cmd[@]}"); then - echo "ERROR: curl request failed: $method $url" >&2 - return 1 - fi - local http_code - http_code=$(echo "$resp" | tail -n1) - local body_lines - body_lines=$(echo "$resp" | sed '$d') - - if [[ "$http_code" -lt 200 || "$http_code" -ge 300 ]]; then - echo "ERROR: HTTP $http_code from $url" >&2 - echo "Response: $body_lines" >&2 - return 1 - fi - - echo "$body_lines" -} - -extract_json() { - local json="$1" - local key="$2" - echo "$json" | jq -r --arg key "$key" 'if type == "object" then (.[$key] // empty) else empty end' 2>/dev/null || true -} - -extract_json_or_default() { - local json="$1" - local key="$2" - local default="$3" - echo "$json" | jq -r --arg key "$key" --arg default "$default" 'if type == "object" then (.[$key] // $default) else $default end' 2>/dev/null || echo "$default" -} - -count_json_collection() { - local json="$1" - local key="${2:-}" - echo "$json" | jq -r --arg key "$key" ' - if type == "array" then length - elif type == "object" then ((.[$key] // []) | if type == "array" then length else 0 end) - else 0 - end - ' 2>/dev/null || echo "0" -} - -json_collection_contains_id() { - local json="$1" - local key="${2:-}" - local id="$3" - echo "$json" | jq -e --arg key "$key" --arg id "$id" ' - (if type == "array" then . elif type == "object" then (.[$key] // []) else [] end) - | map(select(.id == $id)) - | length > 0 - ' >/dev/null 2>&1 -} - -fail() { - echo "ERROR: $*" >&2 - exit 1 -} - -require_command() { - local cmd="$1" - local hint="$2" - if ! command -v "$cmd" >/dev/null 2>&1; then - fail "Required command '$cmd' is not available. $hint" - fi -} - -start_postgres() { - if command -v docker-compose >/dev/null 2>&1 && docker-compose version >/dev/null 2>&1; then - if docker-compose up -d postgres >/dev/null 2>&1; then - echo "docker-compose" - return 0 - fi - fi - if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then - if docker compose up -d postgres >/dev/null 2>&1; then - echo "docker compose" - return 0 - fi - fi - return 1 -} - -wait_for_postgres() { - local compose_cmd="$1" - for i in {1..30}; do - if command -v pg_isready >/dev/null 2>&1; then - if pg_isready -h localhost -p 5432 -U spectraplex >/dev/null 2>&1; then - return 0 - fi - elif $compose_cmd exec -T postgres pg_isready -U spectraplex >/dev/null 2>&1; then - return 0 - fi - sleep 1 - done - return 1 -} - -wait_for_api() { - for _ in {1..30}; do - if curl -sf "$API_URL/health" >/dev/null 2>&1; then - return 0 - fi - - if [[ -n "$API_PID" ]] && ! kill -0 "$API_PID" 2>/dev/null; then - return 2 - fi - - sleep 1 - done - - return 1 -} - -require_command curl "Install curl to call the local API." -require_command cargo "Install Rust/Cargo to build spectraplex-api." -require_command jq "Install jq for JSON response parsing in the smoke script." - -echo "=== Spectraplex Smoke Test ===" -echo "" - -# --------------------------------------------------------------------------- -# 1. Start Postgres -# --------------------------------------------------------------------------- -echo "[1/10] Starting Postgres..." -cd "$PROJECT_DIR" -COMPOSE_CMD="$(start_postgres)" || fail "Could not start Postgres with either 'docker-compose up -d postgres' or 'docker compose up -d postgres'. Check Docker/Compose availability and local container logs." -echo " Postgres started via $COMPOSE_CMD" - -if ! wait_for_postgres "$COMPOSE_CMD"; then - fail "Postgres did not become ready on localhost:5432 within 30 seconds. Check docker compose logs and local port usage." -fi - -# --------------------------------------------------------------------------- -# 2. Build and start the API server -# --------------------------------------------------------------------------- -echo "[2/10] Building API server..." -cd "$PROJECT_DIR" -cargo build --bin spectraplex-api --quiet - -echo " Starting API server..." -export SPECTRAPLEX_CONFIG="$SCRIPT_DIR/smoke-config.toml" -"$PROJECT_DIR/target/debug/spectraplex-api" & -API_PID=$! - -cleanup() { - echo "" - echo "=== Cleaning up ===" - if [[ -n "$API_PID" ]] && kill -0 "$API_PID" 2>/dev/null; then - kill "$API_PID" 2>/dev/null || true - wait "$API_PID" 2>/dev/null || true - fi - echo " Done." -} -trap cleanup EXIT - -if wait_for_api; then - echo " API running on $API_URL (PID $API_PID)" -else - status=$? - if [[ "$status" -eq 2 ]]; then - fail "API process exited before $API_URL/health became healthy." - fi - fail "API did not become healthy at $API_URL/health within 30 seconds." -fi - -# --------------------------------------------------------------------------- -# 3. Create a tenant-scoped API key -# --------------------------------------------------------------------------- -echo "[3/10] Creating tenant API key..." -CREATE_RESP=$(curl_json POST "$API_URL/v1/api-keys" "$LEGACY_KEY" '{"name":"smoke-test-key"}') -API_KEY_ID=$(extract_json "$CREATE_RESP" "id") -TENANT_KEY=$(extract_json "$CREATE_RESP" "key") -if [[ -z "$API_KEY_ID" || -z "$TENANT_KEY" ]]; then - echo "ERROR: Could not parse API key ID/key from response: $CREATE_RESP" - exit 1 -fi -echo " Created tenant key: ${TENANT_KEY:0:12}..." - -# --------------------------------------------------------------------------- -# 4. Register a wallet target -# --------------------------------------------------------------------------- -echo "[4/10] Registering wallet target..." -TARGET_RESP=$(curl_json POST "$API_URL/v1/targets" "$TENANT_KEY" \ - "{\"kind\":\"wallet\",\"network\":\"$TARGET_NETWORK\",\"address\":\"$TARGET_WALLET\",\"mode\":\"both\"}") -TARGET_ID=$(extract_json "$TARGET_RESP" "id") -if [[ -z "$TARGET_ID" ]]; then - echo "ERROR: Could not parse target ID from response: $TARGET_RESP" - exit 1 -fi -echo " Registered target: $TARGET_ID" - -# --------------------------------------------------------------------------- -# 5. Trigger ingestion -# --------------------------------------------------------------------------- -echo "[5/10] Triggering ingestion..." -if [[ "$SKIP_INGEST" == true ]]; then - echo " SKIPPED (--skip-ingest): skipping provider-dependent ingestion" - JOB_ID="" -else - INGEST_RESP=$(curl_json POST "$API_URL/v1/ingest" "$TENANT_KEY" \ - "{\"wallet\":\"$TARGET_WALLET\",\"network\":\"$TARGET_NETWORK\"}") - JOB_ID=$(extract_json "$INGEST_RESP" "id") - if [[ -z "$JOB_ID" ]]; then - echo "ERROR: Could not parse job ID from ingest response: $INGEST_RESP" - exit 1 - fi - echo " Ingest job enqueued: $JOB_ID" -fi - -# --------------------------------------------------------------------------- -# 6. Poll job status -# --------------------------------------------------------------------------- -echo "[6/10] Polling job status..." -if [[ -n "$JOB_ID" ]]; then - for i in {1..30}; do - JOB_STATUS=$(curl_json GET "$API_URL/v1/jobs/$JOB_ID" "$TENANT_KEY") - STATUS=$(extract_json_or_default "$JOB_STATUS" "state" "unknown") - echo " Job status: $STATUS" - if [[ "$STATUS" == "completed" || "$STATUS" == "failed" ]]; then - break - fi - sleep 2 - done - if [[ "$STATUS" == "failed" ]]; then - echo "ERROR: Ingest job $JOB_ID failed" - exit 1 - fi - if [[ "$STATUS" != "completed" ]]; then - echo "ERROR: Ingest job $JOB_ID did not complete within poll window (last status: $STATUS)" - exit 1 - fi -else - echo " Skipped (no job ID)" -fi - -# --------------------------------------------------------------------------- -# 7. Query dataset records (tenant-scoped via target_id) -# --------------------------------------------------------------------------- -echo "[7/10] Querying token_transfers dataset (tenant-scoped)..." -DATASET_RESP=$(curl_json GET "$API_URL/v1/datasets/token_transfers/records?target_id=$TARGET_ID&limit=10" "$TENANT_KEY") -RECORD_COUNT=$(count_json_collection "$DATASET_RESP" "records") -echo " Records returned: $RECORD_COUNT" - -# --------------------------------------------------------------------------- -# 8. Create an export job (tenant-scoped via target_id) -# --------------------------------------------------------------------------- -echo "[8/10] Creating dataset export job (tenant-scoped)..." -EXPORT_RESP=$(curl_json POST "$API_URL/v1/export/dataset" "$TENANT_KEY" \ - "{\"dataset\":\"token_transfers\",\"format\":\"jsonl\",\"target_id\":\"$TARGET_ID\",\"network\":\"$TARGET_NETWORK\",\"sink\":{\"sink_type\":\"local_file\",\"file_path\":\"smoke-export.jsonl\"}}") -EXPORT_JOB_ID=$(extract_json "$EXPORT_RESP" "id") -if [[ -z "$EXPORT_JOB_ID" ]]; then - echo "ERROR: Could not parse export job ID from response: $EXPORT_RESP" - exit 1 -fi -echo " Export job created: $EXPORT_JOB_ID" - -# --------------------------------------------------------------------------- -# 9. Verify API key lifecycle -# --------------------------------------------------------------------------- -echo "[9/10] Verifying API key lifecycle..." -LIST_RESP=$(curl_json GET "$API_URL/v1/api-keys" "$TENANT_KEY") -KEY_COUNT=$(count_json_collection "$LIST_RESP") -echo " Listed $KEY_COUNT active key(s)" -if [[ "$KEY_COUNT" -lt 1 ]]; then - fail "Expected at least one active API key in list response." -fi -if ! json_collection_contains_id "$LIST_RESP" "" "$API_KEY_ID"; then - fail "Created API key $API_KEY_ID was not returned by the API key list endpoint." -fi - -# --------------------------------------------------------------------------- -# 10. Verify target is listable -# --------------------------------------------------------------------------- -echo "[10/10] Verifying target listing..." -TARGETS_RESP=$(curl_json GET "$API_URL/v1/targets?limit=10" "$TENANT_KEY") -TARGET_COUNT=$(count_json_collection "$TARGETS_RESP" "targets") -echo " Listed $TARGET_COUNT target(s)" -if [[ "$TARGET_COUNT" -lt 1 ]]; then - fail "Expected at least one target in list response." -fi -if ! json_collection_contains_id "$TARGETS_RESP" "targets" "$TARGET_ID"; then - fail "Created target $TARGET_ID was not returned by the target list endpoint." -fi - -echo "" -echo "=== Smoke test completed successfully ===" -echo "" -echo "Summary:" -echo " - Tenant API key created and usable" -echo " - Target registration works" -if [[ "$SKIP_INGEST" == true ]]; then - echo " - Ingestion SKIPPED (--skip-ingest)" -else - echo " - Ingestion enqueue works" - echo " - Job status polling works" -fi -echo " - Tenant-scoped dataset queries work (target_id required)" -echo " - Tenant-scoped export job creation works (target_id required)" -echo " - API key listing works" -echo " - Target listing works" +cargo build --locked --bin spectraplex-api +exec python3 scripts/smoke_test.py "$@" diff --git a/scripts/smoke_test.py b/scripts/smoke_test.py new file mode 100644 index 0000000..b0169d9 --- /dev/null +++ b/scripts/smoke_test.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python3 +"""Black-box release checks using an isolated PostgreSQL DB and local RPC fixtures.""" +import argparse +import csv +import io +import json +import os +from pathlib import Path +import secrets +import socket +import subprocess +import tempfile +import threading +import time +from decimal import Decimal +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.error import HTTPError +from urllib.parse import urlsplit, urlunsplit +from urllib.request import Request, urlopen +from provider_fixtures import evm_response + +ROOT = Path(__file__).resolve().parent.parent +SAMPLES = json.loads((ROOT / 'api/static/samples.json').read_text()) + + +def check(condition, message): + if not condition: + raise AssertionError(message) + + +def psql(url, sql): + return subprocess.check_output(['psql', url, '-X', '-v', 'ON_ERROR_STOP=1', '-Atc', sql], text=True).strip() + + +def wait_for(action, description, timeout=60): + end = time.monotonic() + timeout + while time.monotonic() < end: + result = action() + if result: + return result + time.sleep(.2) + raise AssertionError(f'Timed out: {description}') + + +class Provider(BaseHTTPRequestHandler): + errors = [] + evm_chain_id = 1 + evm_head = 14 + evm_blocks = [] + evm_pause = None + evm_in_request = threading.Event() + hl_stage = 0 + + def log_message(self, *_): + pass + + def do_POST(self): + request = json.loads(self.rfile.read(int(self.headers['Content-Length']))) + if request.get('type') in ['userFillsByTime', 'userFunding', 'userNonFundingLedgerUpdates']: + kind = {'userFillsByTime': 'fill', 'userFunding': 'funding', 'userNonFundingLedgerUpdates': 'ledger_update'}[request['type']] + result = [r['raw_metadata']['data'] for r in SAMPLES['hyperliquid']['records'] + if r['raw_metadata']['type'] == kind and r['raw_metadata']['data']['time'] >= request['startTime']] + if self.hl_stage == 0: + result = [r for r in result if kind == 'ledger_update' or (kind == 'fill' and r['dir'].startswith('Open'))] + if kind == 'funding': + result = [{'time': r['time'], 'hash': r.get('hash'), 'delta': {**r, 'type': 'funding'}} for r in result] + elif request.get('type') == 'fundingHistory': + result = [{'coin': request['coin'], 'time': 1735689600000 + i * 3600000, + 'fundingRate': '0.0001', 'premium': '0.00005'} for i in range(2)] + else: + method = request.get('method') + if method and method.startswith('eth_'): + if method == 'eth_getBlockByNumber': + self.evm_blocks.append(int(request['params'][0], 16)) + pause = self.evm_pause + if pause is not None: + self.evm_in_request.set() + if not pause.wait(30): + self.errors.append('Paused fixture was not released') + result = evm_response(method, request.get('params', []), SAMPLES['ethereum']['wallet'], self.evm_chain_id, self.evm_head) + elif method == 'getSignaturesForAddress': + result = [{'signature': signature, 'slot': 300000001, 'err': None, + 'memo': None, 'blockTime': 1735693200, 'confirmationStatus': 'finalized'} + for signature in SIGNATURES] + if request['params'][1].get('until') or request['params'][1].get('before'): + result = [] + elif method == 'getTransaction': + if request['params'][1].get('maxSupportedTransactionVersion') != 0: + self.errors.append('Solana request omitted maxSupportedTransactionVersion=0') + index = SIGNATURES.index(request['params'][0]) + result = SAMPLES['solana']['records'][index]['raw_metadata'] + else: + self.errors.append(f'Unexpected provider request: {method}') + result = None + result = {'jsonrpc': '2.0', 'id': request.get('id'), 'result': result} + data = json.dumps(result).encode() + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(data))) + self.end_headers() + try: + self.wfile.write(data) + except (BrokenPipeError, ConnectionResetError): + pass # Expected when the crash/restart test cancels a provider call. + + +def b58(data): + alphabet = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' + number = int.from_bytes(data, 'big') + value = '' + while number: + number, remainder = divmod(number, 58) + value = alphabet[remainder] + value + return value + + +SIGNATURES = [b58(bytes([i]) * 64) for i in [1, 2]] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--browser', action='store_true') + args = parser.parse_args() + admin_url = os.environ.get('TEST_DATABASE_URL') or os.environ.get('DATABASE_URL') or 'postgres://spectraplex:spectraplex@localhost:5432/spectraplex' + db_name = 'spx_smoke_' + secrets.token_hex(6) + parsed = urlsplit(admin_url) + db_url = urlunsplit((parsed.scheme, parsed.netloc, '/' + db_name, parsed.query, '')) + psql(admin_url, f'CREATE DATABASE {db_name}') + provider = ThreadingHTTPServer(('127.0.0.1', 0), Provider) + threading.Thread(target=provider.serve_forever, daemon=True).start() + api_process = None + with tempfile.TemporaryDirectory(prefix='spectraplex-smoke-') as temp: + temp = Path(temp) + with socket.socket() as sock: + sock.bind(('127.0.0.1', 0)) + port = sock.getsockname()[1] + base = f'http://127.0.0.1:{port}' + admin = secrets.token_hex(32) + config = temp / 'config.toml' + provider_url = f'http://127.0.0.1:{provider.server_port}' + config.write_text(f'''[networks.solana-mainnet] +enabled = true +[networks.hypercore-mainnet] +enabled = true +[networks.ethereum-mainnet] +enabled = true +[[providers]] +network = "ethereum-mainnet" +kind = "rpc" +url = "{provider_url}" +capabilities = ["historical", "logs"] +[[providers]] +network = "solana-mainnet" +kind = "rpc" +url = "{provider_url}" +capabilities = ["historical"] +[[providers]] +network = "hypercore-mainnet" +kind = "rest" +url = "{provider_url}" +capabilities = ["historical"] +''') + env = {**os.environ, 'DATABASE_URL': db_url, 'SPECTRAPLEX_API_KEY': admin, + 'SPECTRAPLEX_CONFIG': str(config), 'SPECTRAPLEX_HOST': '127.0.0.1', + 'SPECTRAPLEX_PORT': str(port), 'SPECTRAPLEX_INGEST_LIMIT': '50', + 'EXPORT_DIR': str(temp / 'exports'), 'SPECTRAPLEX_ENABLE_V1_COMPAT_WRITES': 'false'} + log = (temp / 'api.log').open('w+') + + def call(path, key=admin, data=None, method=None, status=200, raw=False): + request = Request(base + path, data=json.dumps(data).encode() if data is not None else None, + headers={**({'Authorization': f'Bearer {key}'} if key else {}), 'Content-Type': 'application/json'}, + method=method or ('POST' if data is not None else 'GET')) + try: + response = urlopen(request, timeout=15) + except HTTPError as error: + response = error + body = response.read() + check(response.status == status, f'{path}: expected {status}, got {response.status}: {body[:1000]!r}') + if raw: + return body + return json.loads(body) if body else None + + def start(): + nonlocal api_process + api_process = subprocess.Popen([str(ROOT / 'target/debug/spectraplex-api')], cwd=temp, env=env, stdout=log, stderr=log) + def ready(): + if api_process.poll() is not None: + raise AssertionError('API exited during startup') + try: + return call('/ready', key=None)['status'] == 'ready' + except OSError: + return False + wait_for(ready, 'API readiness') + + def stop(): + if api_process and api_process.poll() is None: + api_process.terminate() + try: + api_process.wait(timeout=10) + except subprocess.TimeoutExpired: + api_process.kill() + api_process.wait(timeout=5) + raise AssertionError('API failed graceful SIGTERM shutdown') + + def job_done(job_id, key, export=False): + path = f'/v1/export/jobs/{job_id}' if export else f'/v1/jobs/{job_id}' + def poll(): + result = call(path, key) + check(result['state'] != 'failed', f'Job failed: {result}') + return result if result['state'] == 'completed' else False + return wait_for(poll, path) + + def records(dataset, target, key): + return call(f'/v1/datasets/{dataset}/records?target_id={target}&limit=1000', key) + + def export(dataset, target, key, fmt='jsonl'): + job = call('/v1/export/dataset', key, {'dataset': dataset, 'target_id': target, 'format': fmt}, status=202) + result = job_done(job['id'], key, True) + data = call(f"/v1/export/jobs/{job['id']}/download", key, raw=True) + provenance = call(f"/v1/export/jobs/{job['id']}/download?provenance=true", key) + check(provenance['record_count'] == result['record_count'], 'Provenance count mismatch') + parsed_data = [json.loads(line) for line in data.splitlines()] if fmt == 'jsonl' else list(csv.DictReader(io.StringIO(data.decode()))) + check(len(parsed_data) == result['record_count'], 'Artifact row count mismatch') + return job, parsed_data + + try: + start() + check(call('/ready')['version'] == '1.0.0', 'Wrong release version') + check(b'Data workbench' in call('/', key=None, raw=True), 'Workbench missing') + for path in ['/v1/session', '/v1/jobs', '/v1/targets']: + call(path, key=None, status=401) + tenant = call('/v1/api-keys', data={'name': 'Smoke tenant'}, status=201) + tenant2 = call('/v1/api-keys', data={'name': 'Other tenant'}, status=201) + key, other = tenant['key'], tenant2['key'] + targets = {} + for chain, sample in SAMPLES.items(): + imported = call(f'/v1/demo/{chain}', key, {}, status=200) + targets[chain] = imported['target_id'] + job_done(imported['materialization_job_id'], key) + check(len(records('raw_transactions', imported['target_id'], key)) == len(sample['records']), f'{chain} raw rows missing') + ledger = records('wallet_ledger', imported['target_id'], key) + check(len(ledger) >= 2, f'{chain} ledger is empty or incomplete: {ledger}') + export('wallet_ledger', imported['target_id'], key, 'csv') + export('raw_transactions', imported['target_id'], key) + sol_ledger = records('wallet_ledger', targets['solana'], key) + check(sum(Decimal(str(r['amount'])) for r in sol_ledger if r['entry_type'] == 'fee') == Decimal('-0.000005'), 'Solana transfer was misclassified as fee') + hl_ledger = records('wallet_ledger', targets['hyperliquid'], key) + check(sum(Decimal(str(r['amount'])) for r in hl_ledger if r['asset_symbol'] == 'USDC') == Decimal('1100.45'), 'HL settlement was duplicated or fees/PnL omitted') + check(all(r['timestamp'] < 2_000_000_000 for r in hl_ledger), 'Gold timestamps must use seconds') + pnl = records('hl_pnl_summary', targets['hyperliquid'], key) + check(len(pnl) == 1 and Decimal(str(pnl[0]['net_pnl'])) == Decimal('100.45'), 'HL net PnL mismatch') + print('PASS samples → Bronze → Silver/Gold → CSV/JSONL + provenance, with exact financial fixtures') + + sol = targets['solana'] + balance_before = records('balance_history', sol, key) + replay = call('/v1/demo/solana', key, {}) + job_done(replay['materialization_job_id'], key) + balance_after = records('balance_history', sol, key) + stable = lambda rows: sorted((r['asset_symbol'], r['timestamp'], str(r['balance'])) for r in rows) + check(stable(balance_before) == stable(balance_after), 'Replay changed balances') + check(sum(Decimal(str(r['amount'])) for r in records('wallet_ledger', targets['ethereum'], key)) == Decimal('175'), 'ERC20 ledger expected +250 -75 USDC') + for chain in ['ethereum', 'hyperliquid']: + before = stable(records('balance_history', targets[chain], key)) + again = call(f'/v1/demo/{chain}', key, {}) + job_done(again['materialization_job_id'], key) + check(stable(records('balance_history', targets[chain], key)) == before, f'{chain} replay changed balances') + print('PASS three-chain replay idempotency and representative numeric results') + + for path in [f'/v1/targets/{sol}', f'/v1/datasets/wallet_ledger/records?target_id={sol}']: + call(path, other, status=403) + check(call('/v1/jobs', other) == [], 'Jobs leaked to another tenant') + check(call('/v1/targets', other) == [], 'Targets leaked to another tenant') + call(f'/v1/targets/{sol}/import', key, {'records': SAMPLES['solana']['records']}, status=403) + second = call('/v1/demo/solana', other, {}) + job_done(second['materialization_job_id'], other) + # Replaying a previous run must still see its immutable inputs after another target ingests them. + renormalize = call('/v1/normalize', key, {'wallet': SAMPLES['solana']['wallet'], 'ingestion_run_id': replay['ingestion_run_id']}) + job_done(renormalize['id'], key) + check(stable(records('balance_history', sol, key)) == stable(balance_before), 'Shared transaction changed original history') + print('PASS tenant boundaries and immutable run inputs') + + registered = call('/v1/targets', data={'kind': 'wallet', 'network': 'ethereum-mainnet', 'address': SAMPLES['ethereum']['wallet'], 'mode': 'backfill'}, status=201) + call(f"/v1/targets/{registered['id']}/import", data={'records': []}, status=400) + call(f"/v1/targets/{registered['id']}/import", data={'records': [{'tx_hash': 'invalid', 'timestamp': 1, 'raw_metadata': {}}]}, status=400) + malformed = json.loads(json.dumps(SAMPLES['ethereum']['records'][0])) + malformed['raw_metadata']['data'] = '0xNOTHEX' + call(f"/v1/targets/{registered['id']}/import", data={'records': [malformed]}, status=400) + malformed['raw_metadata'] = {'logs': [None]} + call(f"/v1/targets/{registered['id']}/import", data={'records': [malformed]}, status=400) + imported = call(f"/v1/targets/{registered['id']}/import", data={'records': SAMPLES['ethereum']['records']}) + job_done(imported['materialization_job_id'], admin) + check(len(records('token_transfers', registered['id'], admin)) == 2, 'Admin import did not materialize') + older = json.loads(json.dumps(SAMPLES['ethereum']['records'][0])) + older['tx_hash'] = '0x' + '9' * 64 + older['timestamp'] -= 86400 + older['raw_metadata']['data'] = f'0x{50 * 1000000:064x}' + backfill = call(f"/v1/targets/{registered['id']}/import", data={'records': [older]}) + job_done(backfill['materialization_job_id'], admin) + balances = records('balance_history', registered['id'], admin) + check(len(balances) == 3 and Decimal(str(max(balances, key=lambda r: r['timestamp'])['balance'])) == Decimal('225'), 'Older arrival did not recompute balance suffix') + self_transfer = json.loads(json.dumps(older)) + self_transfer['tx_hash'] = '0x' + '8' * 64 + self_transfer['timestamp'] += 200000 + self_transfer['raw_metadata']['topics'][1] = self_transfer['raw_metadata']['topics'][2] + self_job = call(f"/v1/targets/{registered['id']}/import", data={'records': [self_transfer]}) + job_done(self_job['materialization_job_id'], admin) + check(len(records('wallet_ledger', registered['id'], admin)) == 3, 'Self-transfer created spurious ledger income/expense') + unknown = json.loads(json.dumps(older)) + unknown['tx_hash'] = '0x' + '6' * 64 + unknown['raw_metadata']['address'] = '0x' + 'a' * 40 + unknown_job = call(f"/v1/targets/{registered['id']}/import", data={'records': [unknown]}) + job_done(unknown_job['materialization_job_id'], admin) + unknown_rows = [r for r in records('token_transfers', registered['id'], admin) if r['token_address'] == unknown['raw_metadata']['address']] + check(len(unknown_rows) == 1 and unknown_rows[0]['decimals'] == -1 and Decimal(str(unknown_rows[0]['amount'])) == Decimal('50000000'), 'Unknown ERC20 decimals were guessed') + check(len(records('wallet_ledger', registered['id'], admin)) == 3, 'Unknown units entered financial totals') + # Bronze is immutable: this is a second observed transaction, supplied with metadata. + unknown['tx_hash'] = '0x' + '5' * 64 + unknown['raw_metadata']['token_decimals'] = {unknown['raw_metadata']['address']: 6} + known_job = call(f"/v1/targets/{registered['id']}/import", data={'records': [unknown]}) + job_done(known_job['materialization_job_id'], admin) + known_rows = [r for r in records('wallet_ledger', registered['id'], admin) if r['asset_symbol'] == unknown['raw_metadata']['address']] + check(len(known_rows) == 1 and Decimal(str(known_rows[0]['amount'])) == 50, 'Explicit token decimals were ignored') + print('PASS validated import, unknown-decimal safety, self-transfer neutrality and out-of-order balances') + + call('/v1/targets', key, {'kind': 'wallet', 'network': 'ethereum-mainnet', 'address': SAMPLES['ethereum']['wallet'], 'filter_spec': {'from_block': 1}}, status=201) + evm = call('/v1/ingest', key, {'wallet': SAMPLES['ethereum']['wallet'], 'network': 'ethereum-mainnet'}) + job_done(evm['id'], key) + check(min(Provider.evm_blocks) == 1, 'EVM ignored requested starting block') + evm_target = next(t['id'] for t in call('/v1/targets', key) if t['network'] == 'ethereum-mainnet') + def evm_materialized(): + jobs = call(f'/v1/jobs?target_id={evm_target}', key) + failed = [j for j in jobs if j['state'] == 'failed'] + check(not failed, f'EVM materialization failed: {failed}') + return any(j['kind'] == 'materialize' and j['state'] == 'completed' for j in jobs) + wait_for(evm_materialized, 'EVM materialization') + eraw = records('raw_transactions', evm_target, key) + check(len(eraw) == 1 and len(eraw[0]['raw_metadata']['logs']) == 3 and eraw[0]['timestamp'] == 1735689612, 'EVM lost logs or timestamp') + transfers = records('token_transfers', evm_target, key) + check(len(transfers) == 3 and sum(Decimal(str(r['amount'])) for r in transfers if r['token_symbol'] == 'USDC') == 30, 'Multi-log EVM transfer loss') + custom = next(r for r in transfers if r['token_address'] == '0x' + 'a' * 40) + check(custom['decimals'] == 6 and Decimal(str(custom['amount'])) == 12, 'Provider token metadata was not applied') + eledger = records('wallet_ledger', evm_target, key) + # Wallet-scoped Gold includes other indexed records for this public wallet. + native = [r for r in eledger if r['asset_symbol'] == 'ETH'] + check(sum(Decimal(str(r['amount'])) for r in native) == Decimal('-1.000021'), 'EVM value or fee incorrect') + Provider.evm_head += 2 + Provider.evm_blocks.clear() + empty = call(f'/v1/targets/{evm_target}/ingest', key, {}) + job_done(empty['id'], key) + check(min(Provider.evm_blocks) == 3, 'EVM rescanned the initial range instead of resuming its checkpoint') + checkpoint = psql(db_url, f"SELECT cursor->>'last_block' FROM checkpoints WHERE target_id='{evm_target}' AND source='rpc'") + check(checkpoint == '4', f'Empty EVM window did not advance checkpoint: {checkpoint}') + Provider.evm_chain_id = 2 + mismatch = call(f'/v1/targets/{evm_target}/ingest', key, {'mode': 'backfill'}) + rejected = wait_for(lambda: (r if (r := call(f"/v1/jobs/{mismatch['id']}", key))['state'] == 'failed' else False), 'EVM chain mismatch rejection') + check('chain ID mismatch' in rejected['message'], f'Unexpected EVM failure: {rejected}') + check(psql(db_url, f"SELECT cursor->>'last_block' FROM checkpoints WHERE target_id='{evm_target}' AND source='rpc'") == '4', 'Failed ingestion advanced checkpoint') + Provider.evm_chain_id = 1 + print('PASS EVM multi-log/value/gas ingestion, bounded windows, empty checkpoints and wrong-chain rejection') + + hl_first = call('/v1/ingest', key, {'wallet': SAMPLES['hyperliquid']['wallet'], 'network': 'hypercore-mainnet'}) + job_done(hl_first['id'], key) + hl_target = next(t['id'] for t in call('/v1/targets', key) if t['network'] == 'hypercore-mainnet') + def hl_materializations(count): + jobs = [j for j in call(f'/v1/jobs?target_id={hl_target}', key) if j['kind'] == 'materialize'] + check(not any(j['state'] == 'failed' for j in jobs), f'HL materialization failed: {jobs}') + return jobs if len(jobs) >= count and all(j['state'] == 'completed' for j in jobs) else False + wait_for(lambda: hl_materializations(1), 'first HL batch') + Provider.hl_stage = 1 + hl_second = call(f'/v1/targets/{hl_target}/ingest', key, {}) + job_done(hl_second['id'], key) + hl_jobs = wait_for(lambda: hl_materializations(2), 'second HL batch') + hl_summary = records('hl_pnl_summary', hl_target, key) + check(len(hl_summary) == 1 and Decimal(str(hl_summary[0]['net_pnl'])) == Decimal('100.45') and hl_summary[0]['fill_count'] == 2, 'HL analytics depended on ingestion batch boundaries') + hl_trades = records('hl_trade_history', hl_target, key) + check(len(hl_trades) == 1 and hl_trades[0]['num_fills'] == 2 and hl_trades[0]['closed_at'] < 2_000_000_000, 'HL trade missed its earlier opening fill or has wrong timestamp units') + replay_hl = call('/v1/normalize', key, {'wallet': SAMPLES['hyperliquid']['wallet'], 'ingestion_run_id': hl_jobs[0]['ingestion_run_id']}) + job_done(replay_hl['id'], key) + check([r['id'] for r in records('hl_trade_history', hl_target, key)] == [r['id'] for r in hl_trades], 'HL replay forked trades') + check(len(records('hl_pnl_summary', hl_target, key)) == 1, 'HL replay duplicated PnL') + print('PASS HL nested funding, independent cursors, multi-batch analytics and stable replay') + + live = call('/v1/ingest', key, {'wallet': SAMPLES['solana']['wallet'], 'network': 'solana-mainnet'}) + job_done(live['id'], key) + market = call('/v1/targets', key, {'kind': 'market', 'network': 'hypercore-mainnet', 'address': 'ETH', 'mode': 'backfill'}, status=201) + fetch = call(f"/v1/targets/{market['id']}/ingest", key, {'mode': 'backfill'}) + job_done(fetch['id'], key) + check(len(records('raw_transactions', market['id'], key)) == 2, 'Non-wallet provider path returned no raw records') + export('raw_transactions', market['id'], key, 'csv') + check(not Provider.errors, str(Provider.errors)) + print('PASS local-provider Solana v0 and non-wallet market ingestion') + + # Stop during a real provider request, not merely after completed work. + Provider.evm_head = 18 + Provider.evm_pause = threading.Event() + recovering = call(f'/v1/targets/{evm_target}/ingest', key, {}) + check(Provider.evm_in_request.wait(10), 'Ingestion never reached paused provider') + check(call(f"/v1/jobs/{recovering['id']}", key)['state'] == 'running', 'Expected a genuinely running ingestion') + pending = call('/v1/export/dataset', key, {'dataset': 'wallet_ledger', 'target_id': sol, 'format': 'jsonl'}, status=202) + stop() + Provider.evm_pause.set() + Provider.evm_pause = None + check(psql(db_url, f"SELECT cursor->>'last_block' FROM checkpoints WHERE target_id='{evm_target}' AND source='rpc'") == '4', 'Shutdown committed unfinished ingestion') + # Advance lease age deterministically instead of sleeping five minutes. + psql(db_url, f"UPDATE ingestion_job_attempts SET heartbeat_at=NOW()-INTERVAL '1 hour' WHERE job_id='{recovering['id']}'") + start() + job_done(recovering['id'], key) + check(psql(db_url, f"SELECT MAX(attempt_num) FROM ingestion_job_attempts WHERE job_id='{recovering['id']}'") == '2', 'Restart did not reclaim abandoned work') + check(psql(db_url, f"SELECT cursor->>'last_block' FROM checkpoints WHERE target_id='{evm_target}' AND source='rpc'") == '6', 'Recovered ingestion did not resume from its durable checkpoint') + job_done(pending['id'], key, True) + check(call(f"/v1/export/jobs/{pending['id']}/download", key, raw=True), 'Restart lost export') + call(f"/v1/export/jobs/{pending['id']}/download", other, status=403) + check(stable(records('balance_history', sol, key)) == stable(balance_before), 'Restart changed data') + check(not Provider.errors, str(Provider.errors)) + print('PASS in-flight restart/lease recovery, pending export and durable downloads') + + if args.browser: + subprocess.run(['node', str(ROOT / 'web-tests/workbench.mjs')], env={**os.environ, 'SPECTRAPLEX_TEST_URL': base, 'SPECTRAPLEX_TEST_KEY': admin}, check=True) + call(f"/v1/api-keys/{tenant2['id']}", other, method='DELETE', status=204) + call('/v1/session', other, status=401) + print('PASS key revocation\nAll end-to-end checks passed.') + except Exception: + log.flush() + print('\nAPI log (last 60 lines):') + print('\n'.join((temp / 'api.log').read_text().splitlines()[-60:])) + raise + finally: + stop() + log.close() + provider.shutdown() + psql(admin_url, f'DROP DATABASE {db_name} WITH (FORCE)') + + +if __name__ == '__main__': + main() diff --git a/spectraplex.toml.example b/spectraplex.toml.example index 850335e..bda38c1 100644 --- a/spectraplex.toml.example +++ b/spectraplex.toml.example @@ -6,128 +6,34 @@ log_level = "info" ingest_limit = 50 export_dir = "./exports" -# --------------------------------------------------------------------------- -# Legacy singleton fields (DEPRECATED — prefer structured provider config) -# --------------------------------------------------------------------------- -# These fields still work for backward compatibility. If `[networks.*]` and -# `[[providers]]` sections are present, the legacy fields are ignored. - +# Defaults cover Solana, Ethereum and Hyperliquid. Samples need no providers. +# Public RPCs are rate-limited; use your own endpoints for dependable indexing. solana_rpc_url = "https://api.mainnet-beta.solana.com" -# evm_rpc_url = "https://eth.llamarpc.com" +evm_rpc_url = "https://eth.llamarpc.com" + +# Admin key comes from .env (generated by scripts/setup.sh). +# Optional streaming, trace enrichment and signed job callbacks: # solana_grpc_url = "https://your-yellowstone-endpoint" # solana_grpc_token = "..." -# evm_trace_enabled = false -# evm_trace_rpc_url = "https://eth-mainnet.g.alchemy.com/v2/YOUR_KEY" - -# --------------------------------------------------------------------------- -# Callback signing -# --------------------------------------------------------------------------- -# When set, callback URL payloads (e.g. ingestion and materialization -# job callbacks) are signed with HMAC-SHA256 and delivered with an -# X-Spectraplex-Signature header (sha256=). -# -# callback_hmac_secret = "your-secret-key" - -# Optional: disable best-effort V1 compatibility writes. -# When `false`, API ingestion and streaming workers skip writing to V1 tables -# (`transactions`, `indexer_checkpoints`). Default is `true` for backward -# compatibility. Set to `false` once all consumers are on V2 Bronze/Silver/Gold. -# enable_v1_compat_writes = true - -# --------------------------------------------------------------------------- -# Structured network and provider configuration (V2, recommended) -# --------------------------------------------------------------------------- -# Enable the networks you want to index. Each network needs at least one -# provider entry with the capabilities your workload requires. -# -# When using structured config, API and CLI accept `network` as the primary -# parameter (e.g. "solana-mainnet", "base-mainnet"). The `chain` parameter -# still works as a backward-compatible alias that maps to a default network: -# solana -> solana-mainnet -# ethereum -> ethereum-mainnet -# hyperliquid -> hypercore-mainnet -# -# Adapters are automatically constructed from the provider registry. CLI -# flags like --rpc and --grpc-url override the registry when provided. - -# [networks.solana-mainnet] -# enabled = true -# -# [networks.ethereum-mainnet] -# enabled = true -# +# evm_trace_enabled = true +# evm_trace_rpc_url = "https://your-trace-endpoint" +# callback_hmac_secret = "..." +# enable_v1_compat_writes = false + +# Advanced: structured configuration replaces the defaults when BOTH networks +# and providers are declared. Declare every network you want enabled. +# For example, uncomment these two blocks to enable ONLY Base: # [networks.base-mainnet] # enabled = true -# -# [networks.arbitrum-mainnet] -# enabled = true -# -# [networks.hypercore-mainnet] -# enabled = true - -# -- Solana providers -- - -# [[providers]] -# network = "solana-mainnet" -# kind = "rpc" -# url = "https://api.mainnet-beta.solana.com" -# priority = 1 -# capabilities = ["historical", "balances"] - -# [[providers]] -# network = "solana-mainnet" -# kind = "grpc" -# url = "https://your-yellowstone-endpoint" -# token_env = "SOLANA_GRPC_TOKEN" -# priority = 1 -# capabilities = ["stream"] - -# -- Ethereum providers -- - -# [[providers]] -# network = "ethereum-mainnet" -# kind = "rpc" -# url = "https://eth.llamarpc.com" -# priority = 1 -# capabilities = ["historical", "logs", "receipts", "balances"] - -# [[providers]] -# network = "ethereum-mainnet" -# kind = "trace" -# url = "https://eth-mainnet.g.alchemy.com/v2/YOUR_KEY" -# priority = 2 -# capabilities = ["debug_trace_transaction"] - -# -- Base providers -- - # [[providers]] # network = "base-mainnet" # kind = "rpc" # url = "https://mainnet.base.org" -# priority = 1 # capabilities = ["historical", "logs", "receipts", "balances"] -# -- Arbitrum providers -- - -# [[providers]] -# network = "arbitrum-mainnet" -# kind = "rpc" -# url = "https://arb1.arbitrum.io/rpc" -# priority = 1 -# capabilities = ["historical", "logs", "receipts", "balances"] - -# -- Hyperliquid providers -- - -# [[providers]] -# network = "hypercore-mainnet" -# kind = "rest" -# url = "https://api.hyperliquid.xyz" -# priority = 1 -# capabilities = ["historical"] - -# [[providers]] -# network = "hypercore-mainnet" -# kind = "ws" -# url = "wss://api.hyperliquid.xyz/ws" -# priority = 1 -# capabilities = ["stream"] +# Other network/provider combinations (repeat both blocks for each): +# solana-mainnet / rpc / https://api.mainnet-beta.solana.com / ["historical"] +# ethereum-mainnet / rpc / https://eth.llamarpc.com / ["historical", "logs"] +# arbitrum-mainnet / rpc / https://arb1.arbitrum.io/rpc / ["historical", "logs"] +# hypercore-mainnet / rest / https://api.hyperliquid.xyz / ["historical"] +# hypercore-mainnet / ws / wss://api.hyperliquid.xyz/ws / ["stream"] diff --git a/web-tests/package-lock.json b/web-tests/package-lock.json new file mode 100644 index 0000000..270a3a7 --- /dev/null +++ b/web-tests/package-lock.json @@ -0,0 +1,71 @@ +{ + "name": "spectraplex-browser-checks", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "spectraplex-browser-checks", + "devDependencies": { + "axe-core": "4.11.1", + "playwright": "1.58.2" + } + }, + "node_modules/axe-core": { + "version": "4.11.1", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.11.1.tgz", + "integrity": "sha512-BASOg+YwO2C+346x3LZOeoovTIoTrRqEsqMa6fmfAV0P+U9mFr9NsyOEpiYvFjbc64NMrSswhV50WdXzdb/Z5A==", + "dev": true, + "license": "MPL-2.0", + "engines": { + "node": ">=4" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz", + "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.58.2" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz", + "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + } + } +} diff --git a/web-tests/package.json b/web-tests/package.json new file mode 100644 index 0000000..3cae9f3 --- /dev/null +++ b/web-tests/package.json @@ -0,0 +1,7 @@ +{ + "name": "spectraplex-browser-checks", + "private": true, + "type": "module", + "scripts": { "test": "node workbench.mjs" }, + "devDependencies": { "playwright": "1.58.2", "axe-core": "4.11.1" } +} diff --git a/web-tests/workbench.mjs b/web-tests/workbench.mjs new file mode 100644 index 0000000..7752ed7 --- /dev/null +++ b/web-tests/workbench.mjs @@ -0,0 +1,151 @@ +import { chromium } from 'playwright'; +import assert from 'node:assert/strict'; +import { readFile, mkdir } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; +const require = createRequire(import.meta.url); +const base = process.env.SPECTRAPLEX_TEST_URL; +const key = process.env.SPECTRAPLEX_TEST_KEY; +assert(base && key, 'Set SPECTRAPLEX_TEST_URL and SPECTRAPLEX_TEST_KEY (or run smoke-test.sh --browser)'); +const chrome = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'; +const browser = await chromium.launch({ headless: true, ...(existsSync(chrome) ? { executablePath: chrome } : {}) }); +const page = await browser.newPage({ viewport: { width: 1440, height: 1000 }, acceptDownloads: true }); +// Headless Chromium does not consistently grant clipboard access from a click. +// Scope explicit test permissions to this disposable local deployment. +await page.context().grantPermissions(['clipboard-read', 'clipboard-write'], { origin: new URL(base).origin }); +const errors = []; +page.on('pageerror', error => errors.push(error.message)); +const artifacts = fileURLToPath(new URL('./artifacts/', import.meta.url)); +await mkdir(artifacts, { recursive: true }); +async function accessible() { + await page.evaluate(await readFile(require.resolve('axe-core/axe.min.js'), 'utf8')); + const violations = await page.evaluate(async () => (await window.axe.run(document, { runOnly: { type: 'tag', values: ['wcag2a', 'wcag2aa', 'wcag21aa'] } })).violations); + assert.deepEqual(violations.map(v => `${v.id}: ${v.nodes.map(n => n.target).join(', ')}`), [], 'Accessibility violations'); +} +try { + await page.goto(base); + await page.getByLabel('API key', { exact: true }).fill('incorrect-key'); + await page.getByRole('button', { name: 'Connect', exact: true }).click(); + await page.getByRole('alert').waitFor(); + assert.match(await page.getByRole('alert').innerText(), /API key/); + await page.getByLabel('API key', { exact: true }).fill(key); + await page.getByRole('button', { name: 'Connect', exact: true }).click(); + await page.locator('#workspace').waitFor({ state: 'visible' }); + await page.getByRole('button', { name: 'Solana', exact: true }).click(); + await page.waitForFunction(() => document.querySelector('#selected-target-name').textContent === 'Sample · Solana wallet'); + await page.waitForFunction(() => document.querySelectorAll('#records-table tbody tr').length >= 3, { timeout: 30000 }); + assert(await page.locator('#ingest').isDisabled(), 'Synthetic data must not be fetched from live providers'); + assert(await page.locator('#next').isDisabled(), 'Pagination must stop at end of records'); + await accessible(); + await page.screenshot({ path: `${artifacts}/desktop.png`, fullPage: true }); + + await page.getByLabel('Dataset', { exact: true }).selectOption('raw_transactions'); + await page.waitForFunction(() => document.querySelectorAll('#records-table tbody tr').length === 2); + await page.getByText('View JSON', { exact: true }).first().click(); + assert.match(await page.locator('#records-table').innerText(), /raw_metadata/); + await page.getByLabel('From', { exact: false }).fill('2099-01-01T00:00'); + await page.getByRole('button', { name: 'Apply', exact: true }).click(); + await page.locator('#empty-records').waitFor({ state: 'visible' }); + await page.getByRole('button', { name: 'Reset', exact: true }).click(); + await page.waitForFunction(() => document.querySelectorAll('#records-table tbody tr').length === 2); + await page.getByRole('button', { name: 'Export CSV', exact: true }).click(); + await page.locator('#panel-activity').waitFor({ state: 'visible' }); + await page.waitForFunction(() => document.querySelector('#jobs-table tbody tr')?.textContent.includes('Download'), { timeout: 30000 }); + const downloadEvent = page.waitForEvent('download'); + await page.locator('#jobs-table tbody tr').first().getByRole('button', { name: 'Download', exact: true }).click(); + const download = await downloadEvent; + const csv = await readFile(await download.path(), 'utf8'); + assert.match(csv, /raw_metadata/); assert.match(csv, /sample-solana/); + const provenanceEvent = page.waitForEvent('download'); + await page.locator('#jobs-table tbody tr').first().getByRole('button', { name: 'Provenance', exact: true }).click(); + const provenance = JSON.parse(await readFile(await (await provenanceEvent).path(), 'utf8')); + assert.equal(provenance.record_count, 2); + + // Retry an actual failed provider job without changing its requested mode. + const failed = page.locator('#jobs-table tbody tr').filter({ hasText: 'chain ID mismatch' }); + const retried = page.waitForRequest(r => r.method() === 'POST' && /\/targets\/[^/]+\/ingest$/.test(r.url())); + await failed.getByRole('button', { name: 'Retry', exact: true }).click(); + assert.equal((await retried).postDataJSON().mode, 'backfill'); + + // Exercise target creation, browser file validation, JSONL import and real paging. + await page.getByRole('button', { name: 'Explore data', exact: true }).click(); + await page.getByText('Add a target', { exact: true }).click(); + await page.getByLabel('Network', { exact: true }).selectOption('ethereum-mainnet'); + const wallet = '0x' + '3'.repeat(40); + await page.getByLabel('Address / market symbol', { exact: true }).fill(wallet); + await page.locator('#target-label').fill('Browser import'); + await page.locator('#target-start-block').fill('10'); + const created = page.waitForResponse(r => r.url().endsWith('/v1/targets') && r.request().method() === 'POST'); + await page.getByRole('button', { name: 'Create target', exact: true }).click(); + assert.equal((await (await created).json()).filter_spec.from_block, 10); + await page.waitForFunction(() => document.querySelector('#selected-target-name').textContent === 'Browser import'); + await page.getByLabel('Dataset', { exact: true }).selectOption('raw_transactions'); + await page.locator('#import-section summary').click(); + await page.locator('#import-file').setInputFiles({ name: 'invalid.json', mimeType: 'application/json', buffer: Buffer.from('not json') }); + await page.getByRole('button', { name: 'Import & materialize', exact: true }).click(); + await page.waitForFunction(() => document.querySelector('#notice').textContent.includes('Invalid JSON/JSONL')); + const sample = JSON.parse(await readFile(new URL('../api/static/samples.json', import.meta.url), 'utf8')).ethereum.records[0]; + const records = Array.from({ length: 60 }, (_, i) => { + const record = structuredClone(sample); + record.tx_hash = '0x' + (50000 + i).toString(16).padStart(64, '0'); + record.timestamp = 1735689600 + i; + record.raw_metadata.topics[2] = '0x' + wallet.slice(2).padStart(64, '0'); + return record; + }); + await page.locator('#import-file').setInputFiles({ name: 'history.jsonl', mimeType: 'application/json', buffer: Buffer.from(records.map(r => JSON.stringify(r)).join('\n')) }); + await page.getByRole('button', { name: 'Import & materialize', exact: true }).click(); + await page.waitForFunction(() => document.querySelectorAll('#records-table tbody tr').length === 50); + const firstPage = await page.locator('#records-table tbody tr').allTextContents(); + await page.locator('#next').click(); + await page.waitForFunction(() => document.querySelectorAll('#records-table tbody tr').length === 10); + const secondPage = await page.locator('#records-table tbody tr').allTextContents(); + assert.equal(new Set([...firstPage, ...secondPage]).size, 60, 'Pages overlap or lost imported rows'); + assert(await page.locator('#next').isDisabled()); + await page.locator('#previous').click(); + await page.waitForFunction(() => document.querySelectorAll('#records-table tbody tr').length === 50); + assert.deepEqual(await page.locator('#records-table tbody tr').allTextContents(), firstPage); + + await page.getByRole('button', { name: 'API keys', exact: true }).click(); + await page.getByLabel('Key name', { exact: true }).fill('Browser test'); + await page.getByRole('button', { name: 'Create key', exact: true }).click(); + await page.locator('#new-key-box').waitFor({ state: 'visible' }); + const tenantKey = await page.locator('#new-key').inputValue(); + assert.match(tenantKey, /^spx_/); + await page.bringToFront(); + await page.getByRole('button', { name: 'Copy key', exact: true }).click(); + await page.waitForFunction(() => document.querySelector('#notice').textContent.includes('Key copied')); + assert.equal(await page.evaluate(() => navigator.clipboard.readText()), tenantKey); + await page.getByRole('button', { name: 'Disconnect', exact: true }).click(); + assert.equal(await page.locator('#new-key').inputValue(), ''); + assert.equal(await page.evaluate(() => localStorage.length + sessionStorage.length), 0, 'Keys must not persist in browser storage'); + await page.getByLabel('API key', { exact: true }).fill(tenantKey); + await page.getByRole('button', { name: 'Connect', exact: true }).click(); + await page.locator('#workspace').waitFor({ state: 'visible' }); + await page.locator('#empty-target').waitFor({ state: 'visible' }); + assert.equal(await page.locator('#target-list option').count(), 0, 'New tenant inherited targets'); + await page.getByRole('button', { name: 'Ethereum', exact: true }).click(); + await page.waitForFunction(() => document.querySelectorAll('#records-table tbody tr').length === 2, { timeout: 30000 }); + assert(await page.locator('#import-section').isHidden(), 'Tenant offered arbitrary imports'); + await page.setViewportSize({ width: 390, height: 844 }); + await accessible(); + assert(await page.locator('#records-table tbody tr').evaluateAll(rows => rows.every(row => row.clientHeight < 120)), 'Closed records should not wrap hashes into excessively tall rows'); + await page.screenshot({ path: `${artifacts}/mobile.png`, fullPage: true }); + assert(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth), 'Mobile page overflows horizontally'); + await page.setViewportSize({ width: 320, height: 800 }); + assert(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth), '320px reflow overflows horizontally'); + await accessible(); + await page.getByRole('button', { name: 'API keys', exact: true }).click(); + page.once('dialog', dialog => dialog.accept()); + await page.locator('#keys-table tbody tr').filter({ hasText: 'Browser test' }).getByRole('button', { name: 'Revoke', exact: true }).click(); + await page.locator('#login').waitFor({ state: 'visible' }); + assert.equal(await page.locator('#api-key').inputValue(), '', 'Revocation must clear credentials'); + assert.deepEqual(errors, [], 'Browser runtime errors'); + console.log('PASS browser: authentication, samples, filters, JSONL import, real pagination, retries, downloads, keys, tenant isolation, mobile layout and WCAG AA checks'); +} catch (error) { + console.error('Browser notice:', await page.locator('#notice').innerText().catch(() => '(unavailable)')); + await page.screenshot({ path: `${artifacts}/failure.png`, fullPage: true, mask: [page.locator('#api-key'), page.locator('#new-key')] }).catch(() => {}); + throw error; +} finally { + await browser.close(); +}