feat(telemetry)!: adopt DuckDB 2 and upgrade dependencies

.dockerignore

@@ -11,6 +11,8 @@ data/
 /testdata/
 
 # Build
+/.cache/
+telemetry.test
 bin/
 fanout
 ui/host/node_modules/

.github/workflows/ci.yml

@@ -18,6 +18,25 @@ env:
   CGO_ENABLED: "1"
 
 jobs:
+  duckdb:
+    name: DuckDB 2 (${{ matrix.runner }})
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [ubuntu-24.04, ubuntu-24.04-arm, macos-15, macos-15-intel]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+          cache: true
+      - name: Verify pinned engine, typed Parquet and SQL boundary
+        run: >-
+          bash scripts/with-duckdb.sh go test ./internal/query ./internal/telemetry ./internal/observability ./internal/ingest
+          -run 'TestPinnedEngine|TestSQLBoundary|TestLogicalType|TestVariant|TestUnsupportedFormat|TestCommitBatchAllocation|TestDependencies|TestExportsBound|TestMaximumAttributeNesting|TestHTTPRejectsExcessiveAttributeNesting'
+          -count=1
+
   # Every step below runs a justfile recipe, named after it. Reproduce any
   # failure locally with the command in the step name — nothing about the gate
   # lives only in this file.
@@ -40,7 +59,7 @@ jobs:
       # must be a deliberate commit rather than a surprise from a CI run.
       - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
         with:
-          bun-version: 1.4.0
+          bun-version: 1.4.2
 
       # The documentation site is npm rather than bun: Astro and Starlight are
       # exercised on npm upstream, and the lockfile is what `just site-build`

.github/workflows/codeql.yml

@@ -48,7 +48,7 @@ jobs:
           build-mode: ${{ matrix.build-mode }}
 
       - if: matrix.build-mode == 'manual'
-        run: go build ./...
+        run: bash scripts/with-duckdb.sh go build ./...
 
       - uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
         with:

.github/workflows/release.yml

@@ -13,7 +13,7 @@ permissions:
   contents: read
 
 env:
-  GO_VERSION: "1.27.0"
+  GO_VERSION: "1.27.1"
   IMAGE: ghcr.io/labstack/fanout
   DOCKERHUB_IMAGE: docker.io/labstack/fanout
   MCP_PUBLISHER_VERSION: "1.8.1"
@@ -52,7 +52,7 @@ jobs:
       - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4
       - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
         with:
-          bun-version: 1.4.0
+          bun-version: 1.4.2
       - uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0
         with:
           version: v2.13.1
@@ -88,8 +88,9 @@ jobs:
         run: |
           set -euo pipefail
           version="${GITHUB_REF_NAME}"
+          bash scripts/with-duckdb.sh go test ./internal/query -run 'TestPinnedEngine|TestSQLBoundary' -count=1
           mkdir -p stage
-          go build -trimpath -ldflags "-s -w -X main.version=${version}" -o stage/fanout ./cmd/fanout
+          bash scripts/with-duckdb.sh go build -trimpath -ldflags "-s -w -X main.version=${version}" -o stage/fanout ./cmd/fanout
           for doc in LICENSE NOTICE README.md THIRD_PARTY_NOTICES TRADEMARK.md; do
             if [ -f "${doc}" ]; then cp "${doc}" stage/; fi
           done

.github/workflows/site.yml

@@ -32,6 +32,11 @@ on:
       - "internal/mcp/**"
       - "cmd/fanout/**"
       - "cmd/fanout-docgen/**"
+      - "internal/duckdb/**"
+      - "scripts/with-duckdb.sh"
+      - "scripts/go-build-tags.sh"
+      - "go.mod"
+      - "go.sum"
       - ".github/workflows/site.yml"
   # Publishing the current main on demand, without an empty commit.
   workflow_dispatch:
@@ -77,7 +82,7 @@ jobs:
       - name: Refuse a reference that is behind the code
         env:
           CGO_ENABLED: "1"
-        run: go run ./cmd/fanout-docgen --check
+        run: bash scripts/with-duckdb.sh go run ./cmd/fanout-docgen --check
 
       - name: Install dependencies
         working-directory: site

.gitignore

@@ -1,5 +1,6 @@
 # Binaries
 /bin/
+/.cache/
 *.exe
 
 # Data

AGENTS.md

@@ -16,6 +16,60 @@
   with application tables must have a positive applied Goose version before
   initialization. An empty or zero-only version table is not sufficient.
 
+## Telemetry engine and format
+
+- Use the pinned DuckDB 2 engine in `internal/duckdb`, built with
+  `scripts/with-duckdb.sh`. A Go driver version does not identify its bundled
+  native engine; verify `SELECT version()` when changing the pin.
+- Build and test on native Linux or macOS, AMD64 or ARM64. The wrapper verifies
+  each platform archive's checksum and uses its matching headers and statically
+  linked core, JSON, Parquet, and time-zone extensions. Do not add dynamic
+  extension downloads or a second engine path.
+- Store OTLP attributes and resource attributes as typed maps in canonical rows
+  and shredded Parquet VARIANT columns. Preserve integers, booleans, bytes,
+  nested values, and nanosecond UTC timestamps. Convert nested SQL results to
+  JSON only at the client boundary; attribute keys containing dots are literal.
+- Dashboard file snapshots use each signal's actual event-time footer bounds,
+  including signed nanoseconds. Unknown statistics include the file. Never
+  substitute ingestion time or prune public arbitrary SQL from a dashboard scope.
+- Endpoint histograms, log counts, and notable-trace candidates acknowledge
+  complete immutable batch IDs transactionally. Query cache markers and rows in
+  one read transaction under the pinned file snapshot; uncached active files
+  remain immediately visible. Compaction and retention must invalidate retired
+  contributions without double counting or losing late publications.
+- Version disposable read-cache schema and semantics together; mismatches rebuild
+  all private read tables. Cache minute aggregates, per-batch trace bounds,
+  and one incremental candidate per trace, never individual event copies or
+  four globally rewritten scope indexes. Mixed trace scopes use batch parts. Exact
+  clipped minutes read footer-pruned Parquet, and compaction derives complete
+  output contributions from cached inputs. Body search matches redacted text.
+  Analytical service/edge watermark lag is separate. Completed-batch writes
+  have their own gate and write-pool slot; analytical writes use disjoint tables.
+  Maintenance must hold both gates for checkpointing.
+- Batch format 3, including its physical schema, is the only accepted telemetry
+  format. Schema changes require a format version change. Disable schema unioning
+  and Hive partition inference; use native Parquet binding so VARIANT extracts
+  can reach the scan. Pre-project hot messaging fields into scalar read-view
+  columns; this preview does not push general VARIANT extracts across views.
+  Reject unsupported metadata before cleanup or schema rewriting, preserve those files, and add
+  no legacy reader or format fallback.
+- Keep native `TIMESTAMPTZ_NS` columns in time predicates. Bind Go window
+  parameters as `?::TIMESTAMP_NS::TIMESTAMPTZ_NS` to preserve nanoseconds;
+  cast returned timestamps and datetime-function arguments only. Every engine
+  connection uses UTC. Do not change Parquet UTC-instant semantics for speed.
+- Offline verification reuses one bounded, non-spilling native engine to decode
+  all Parquet columns and checks physical schemas, complete span sort tuples,
+  and exact index ranges. Operational errors and unsupported formats never
+  authorize quarantine. Discarded hashes force decoding; they are not checksums.
+- Keep DuckDB 2's memory-governed asynchronous I/O defaults unless measurements
+  justify tuning them. Do not force an unbounded read-ahead depth.
+- Arbitrary SQL is a single read-only SELECT over approved telemetry relations.
+  Parse its AST and describe/project results on the same connection and pinned
+  Parquet snapshot. Keep engine file access and configuration locked down.
+- Rooted service traversal uses keyed recursion over the complete scoped edge
+  rollup, keeps namespaces separate, and enforces hop, accumulated-node, and
+  execution-time limits. Report truncation explicitly.
+
 ## Product versioning
 
 Fanout uses CalVer with the format `YYYY.M.N[-alpha|-beta|-rc]`.

CONTRIBUTING.md

@@ -10,13 +10,37 @@ just build     # browser assets, then the binaries
 just check     # the full gate
 ```
 
-You need Go and a C compiler with `CGO_ENABLED=1` (DuckDB is a cgo dependency),
+You need Go 1.27.1 and a C/C++ toolchain on native Linux or macOS (amd64 or arm64),
 [Bun](https://bun.sh), [just](https://just.systems),
 [golangci-lint](https://golangci-lint.run/), and
 [Lefthook](https://github.com/evilmartians/lefthook). Running in local auth mode
 also requires a 32-character authentication code secret; SMTP and an AI key are
 optional — see [README](README.md#requirements).
 
+The DuckDB wrapper downloads the pinned, checksum-verified engine from
+[Fanout's dependency artifact mirror](https://github.com/labstack/fanout/releases/tag/duckdb-v2.0.0-alpha43763)
+on the first build. Linux needs the C++ runtime (`libstdc++`); macOS needs
+Xcode Command Line Tools and `libc++`. Cross-compilation is unsupported.
+Use `just` recipes or prefix Go commands with the wrapper:
+
+```sh
+bash scripts/with-duckdb.sh go build ./cmd/fanout
+bash scripts/with-duckdb.sh go test ./internal/ingest
+```
+
+Editor tooling also needs the same build tags, headers, and linker flags.
+Launch a fresh editor process through the wrapper, for example
+`bash scripts/with-duckdb.sh code .`, so its Go tools inherit that environment.
+Alternatively configure your editor to launch `gopls` through
+`bash scripts/with-duckdb.sh gopls serve`. An already-running editor process
+must be restarted to inherit these variables.
+
+The dependency mirror preserves the original DuckDB `v2.0.0-alpha43763`
+archives at upstream commit `96063b9e39` byte for byte. Its `duckdb-` tag
+namespace is separate from Fanout's product CalVer releases; new engine bytes
+require a new artifact tag and checksum pins. It never supplies a second engine
+or downloads extensions at runtime.
+
 ## Before you open a pull request
 
 Run `just check` and `just test-race`. Together they match the CI gate:

Dockerfile

@@ -2,7 +2,7 @@
 
 # Bun is a build compiler only. Neither Bun nor Node is copied into the final
 # image or launched by the Fanout process.
-FROM oven/bun:1.4.0@sha256:5ff609364c049b54eb0ff560ec96319729a972078ef2c755d758f0c6ef89c2d6 AS ui-apps-build
+FROM oven/bun:1.4.2@sha256:9114c058aeae42162ee16dd5084b95fe9473970bb6bcb5b232ab1630f0546895 AS ui-apps-build
 WORKDIR /app
 COPY ui/apps/package.json ui/apps/bun.lock ./ui/apps/
 RUN cd ui/apps && bun install --frozen-lockfile
@@ -11,7 +11,7 @@ COPY ui/apps/ ./ui/apps/
 COPY internal/mcp/apps/ ./internal/mcp/apps/
 RUN cd ui/apps && bun run build
 
-FROM oven/bun:1.4.0@sha256:5ff609364c049b54eb0ff560ec96319729a972078ef2c755d758f0c6ef89c2d6 AS ui-host-build
+FROM oven/bun:1.4.2@sha256:9114c058aeae42162ee16dd5084b95fe9473970bb6bcb5b232ab1630f0546895 AS ui-host-build
 WORKDIR /app
 COPY ui/host/package.json ui/host/bun.lock ./ui/host/
 RUN cd ui/host && bun install --frozen-lockfile
@@ -24,7 +24,7 @@ RUN cd ui/host && bun run build
 # is no cross toolchain here. `--platform=$BUILDPLATFORM` is therefore only
 # correct while TARGETPLATFORM equals BUILDPLATFORM. Adding an architecture to
 # the CI matrix means either a native runner for it or QEMU, not a GOARCH flag.
-FROM --platform=$BUILDPLATFORM golang:1.27-bookworm@sha256:ded31c68586d2e49e760acc2e65a884b23d032e9bbbed0ae0c55abd3fcaf4452 AS build
+FROM --platform=$BUILDPLATFORM golang:1.27.1-bookworm@sha256:69a7b9788769bec032d238959b61854e9ae87f57be9029ec04e9885fabf99195 AS build
 ARG TARGETOS
 ARG TARGETARCH
 ARG VERSION=dev
@@ -37,7 +37,7 @@ COPY --from=ui-host-build /app/internal/ui/dist/ ./internal/ui/dist/
 RUN --mount=type=cache,target=/go/pkg/mod \
     --mount=type=cache,target=/root/.cache/go-build \
     CGO_ENABLED=1 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
-    go build -ldflags="-s -w -X main.version=${VERSION}" -o fanout ./cmd/fanout
+    bash scripts/with-duckdb.sh go build -ldflags="-s -w -X main.version=${VERSION}" -o fanout ./cmd/fanout
 
 # Distroless has no shell with which to create mutable paths. Prepare the data
 # directory here, then copy it with the runtime user's ownership below.

README.md

@@ -33,6 +33,10 @@ browser client, an in-process agent, and any
 external MCP host all reach the same typed observability contract rather than
 issuing raw SQL.
 
+Attribute and resource values support up to 128 nested arrays or objects.
+An export that exceeds this limit is rejected in full with OTLP
+`INVALID_ARGUMENT` (HTTP 400); accepted values retain their original types.
+
 Parquet is authoritative telemetry, DuckDB query state is rebuildable, and
 SQLite is reserved for transactional product state. Native compaction
 prepares replacements while reads continue and briefly gates readers only for
@@ -83,7 +87,10 @@ Fanout.
 
 ## Requirements
 
-- **Go and a C compiler** with `CGO_ENABLED=1` — DuckDB is a cgo dependency
+- **Go 1.27.1 and a C/C++ toolchain** on native Linux or macOS, amd64 or arm64.
+  Build with `just build` or `bash scripts/with-duckdb.sh go build ./cmd/fanout`;
+  the wrapper supplies DuckDB's pinned headers, static libraries, and CGO flags.
+  The first build needs network access to download the checksum-verified engine.
 - **[Bun](https://bun.sh)** — compiles the browser assets
 - **[just](https://just.systems)** — task runner
 - A 32-character **authentication code secret**