diff --git a/.claude/commands/cover.md b/.claude/commands/cover.md index 021def65..a760c242 100644 --- a/.claude/commands/cover.md +++ b/.claude/commands/cover.md @@ -1,6 +1,6 @@ --- description: Render coverage HTML for a suite and surface drops below threshold -argument-hint: [unit|integration|e2e|sdk|merge|all] (default: merge whatever exists) +argument-hint: [unit|integration|e2e|go-sdk|ts-unit|ts-e2e|ts-total|merge|all] (default: merge whatever exists) --- Generate the coverage report and surface anything below threshold from `.testcoverage.yml`. @@ -13,10 +13,11 @@ Behavior: - **unit**: `make test-unit` (gates per-suite + writes `tmp/coverage/unit/`) - **integration**: `make test-integration` (requires Docker) - **e2e**: `make test-e2e` (requires Docker; orchestrator + cover binary) -- **ts-unit**: `make test-ts` (SDK unit tests + coverage + gate against `suites.ts-unit`) +- **go-sdk**: `make test-sdk-go` (nested module `clients/go`; gates against `suites.go-sdk`, rendered separately and never merged into the Go total) +- **ts-unit**: `make test-sdk-ts` (TS SDK unit tests + coverage + gate against `suites.ts-unit`) - **ts-e2e**: emitted as a side effect of `make test-e2e` (the orchestrator always passes `--coverage` to the e2e vitest run; informational only, no standalone gate) - **ts-total**: `make cov` (runs `cov report` — one consolidated Go + TS summary with per-suite HTML links + all gates; fails if *no* suite has data) -- **all**: `make test-all` (all four suites sequentially + `make cov`) +- **all**: `make test-all` (every coverage-producing suite sequentially + `make cov`; `test-sdk-go-e2e` is excluded, it needs a live server it did not start and runs uninstrumented) After the run completes: diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e1c5d695..ba89ab59 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,8 +1,11 @@ version: 2 updates: - # Go modules + # Go modules. TWO directories: Dependabot does not descend into nested + # modules, so clients/go needs its own entry (stdlib-only today). - package-ecosystem: gomod - directory: / + directories: + - / + - /clients/go schedule: interval: weekly day: monday diff --git a/.github/labeler.yml b/.github/labeler.yml index 4df42c01..d164e157 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -43,7 +43,9 @@ - changed-files: - any-glob-to-any-file: - "clients/ts/**" + - "clients/go/**" - "tests/e2e/sdk/**" + - "tests/conformance/**" "area/docs": - changed-files: @@ -83,8 +85,8 @@ dependencies: - changed-files: - any-glob-to-any-file: - - "go.mod" - - "go.sum" + - "**/go.mod" + - "**/go.sum" - "**/package.json" - "**/pnpm-lock.yaml" - "**/package-lock.json" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c0b05f9..943aa504 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -204,8 +204,10 @@ jobs: uses: ./.github/actions/setup-env with: go-cache-suffix: "-unit" - - name: Run Go unit tests + SDK vitest tests - run: make test-unit test-ts COV_DEFER=1 + - name: Run Go unit tests + both SDK suites + run: make test-unit test-sdk COV_DEFER=1 + # One fragment, three suites (unit, ts-unit, go-sdk) — all deferred to + # the `coverage` job. Whole-tree path so a new suite needs no edit here. - name: Upload coverage fragment uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/.testcoverage.yml b/.testcoverage.yml index 669d65be..cc5fa11b 100644 --- a/.testcoverage.yml +++ b/.testcoverage.yml @@ -13,6 +13,7 @@ # coverage number — unit alone covers only `./internal/...` packages # exercised by `*_test.go` files (below total threshold); merged # coverage adds integration- and e2e-only paths and clears it. +# Nested-module suites (go-sdk) are gated separately — see suites.go-sdk. profile: tmp/coverage/total/coverage.txt local-prefix: github.com/Wave-RF/WaveHouse @@ -22,7 +23,7 @@ local-prefix: github.com/Wave-RF/WaveHouse threshold: total: 80 -# Per-suite coverage thresholds (custom key — read by scripts/coverage.sh, +# Per-suite coverage thresholds (custom key — read by scripts/cov, # silently ignored by go-test-coverage). Single source of truth for what # percentage each suite must hit before it counts as passing. # @@ -30,6 +31,25 @@ suites: unit: 80 integration: 20 e2e: 60 + # Go SDK (clients/go) — a NESTED module, so the root module's + # `-coverpkg=./...` can never reach it: rendered and gated on its own, + # never merged into the total above, nothing to add under exclude.paths. + # 75 vs 88.5% measured. The gap is deliberate slack, matching how every other + # floor here is set (ts-unit is 40 against a measured 76%): these are + # regression floors, not targets, so ordinary churn doesn't red the build. + # Raise it when a real regression slips under 75. + # + # ONE key here, three for the TS SDK below, because the TS SDK has two + # coverage-PRODUCING suites and this has one. `ts-unit` is vitest, `ts-e2e` + # is the same SDK source re-measured by the orchestrator's live-server run, + # and `ts-total` exists only because those two have to be merged before a + # meaningful floor can be applied. `make test-sdk-go` is the Go SDK's only + # instrumented suite, so a `go-sdk-total` would be a copy of this number and + # a `go-sdk-e2e` would gate a suite that emits no covdata: `test-sdk-go-e2e` + # drives a server it did not start and runs uninstrumented. Wiring that suite + # into the orchestrator so it earns the same three-gate shape is #518 — add + # the other two keys with it, not before. + go-sdk: 75 # TypeScript SDK suites — see scripts/cov for the merge / render logic. # vitest gates via --coverage.thresholds.statements; the merged ts-total # is gated by `cov ts-merge` against the value below. Tune ts-total diff --git a/AGENTS.md b/AGENTS.md index 9967534a..0596a01d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,7 +60,7 @@ The invariant index — what must stay true. Full narrative and rationale live i 11. **Hasura-style access control: fail-closed (security)** — `policy.IsAdmin` (role == `admin_role`, **exact case-sensitive**, default `"admin"`) is the single admin check, shared by `Evaluate`/`ResolveRole`/`Validate`/the `/v1/ops` gate/`RoleAllowed`. Empty/absent role matches nothing (no `"*"` wildcard); `Validate` rejects empty role keys; a `nil` policy (deleted) denies **everyone incl. admin** via a role — a total lockout for token-based callers, so bootstrap from the policy file, never an implicit admin grant (**exception:** the operator key's `auth.IsOperator` bit passes the `/v1/ops` gate even under a `nil` policy — a deliberate break-glass restore over HTTP, see #7). `default_role` is the one sanctioned roleless exception (`ResolveRole` maps empty → it pre-eval); `default_role == admin_role` is permitted but dev-only and loudly warned (`policy.DefaultRoleGrantsAdmin`). Preserve when touching `internal/policy` (policy twin of #13; see #159). Detail: architecture.md § `policy/`. 12. **Structured queries: column authz fail-closed (security)** — `POST /v1/query?table={table}`: typed AST validated against schema, permission-enforced, timestamp-bucketed for cache, `DefaultMaxRows` (10,000) cap. Every column reference — projection, aggregation args, `filters`, `group_by`, `order_by`, `time_range` — is authorized inside `query.Build` (the single chokepoint that enumerates them all), so no clause can skip the role's `allow_columns`/`deny_columns` check (#223). A `select_all` read by a *column-restricted* role expands to its allowed columns via `policy.AllowedProjection`, never a bare `SELECT *`; *unrestricted*/admin roles keep `SELECT *` (`policy.RestrictsColumns` decides). Omitting `columns` selects nothing (`ErrEmptyProjection` → `200 []`); `["*"]` is the literal column `*` (schema-gated, not a wildcard); a table-granted role with no readable columns fails closed (`ErrNoReadableColumns` → `403`). Structured and live-stream (`stream.filterColumns`) reads share the one per-column decision `policy.IsColumnAllowed`, so column visibility can't drift. Row visibility has the same one-source guarantee (#319): `Evaluate` resolves a role's row-`filter` once (`resolvePredicates`), and both surfaces consume that single resolution — the query path renders it to SQL (`predicatesToSQL`), the stream evaluates it in memory per subscriber (`ResolvedPermissions.RowVisible`, whose type-aware comparison fails closed on anything it can't prove about the ingested payload — `policy.ColumnSpec`, with `DateTime`/`DateTime64` operands compared as instants through the ingest grammar (`discovery.Column.TimeParser`) and claim constants rendered canonically and digit-exact by the one shared rule `policy.CanonicalScalar` (#457 — which also refuses a float64 at/past 2^53 rather than match a neighboring ID, and whose ok=false — an absent claim, a structured value, no canonical form — makes the predicate match no rows on BOTH surfaces: `1 = 0` in SQL, every row withheld in memory); numeric comparison runs in the column's STORAGE domain (`policy.NumericSpec`, classified by `discovery.NumericStorageOf` — Float width rounding, Decimal scale truncation, integer exactness, both operands narrowed as ClickHouse narrows stored value and bound constant, out-of-range operands refused rather than modeled; the `tests/integration` differential oracle holds in-range verdicts equal to a live ClickHouse's and the never-admit-where-SQL-hides direction for the refused out-of-range ones); an event whose insert later fails into the DLQ is the one residual payload-vs-stored asymmetry, documented in the access-control enforcement caution) — so row visibility can't drift either. Preserve when touching `internal/query` or the structured-query handler. Detail: architecture.md § `query/`. 13. **Named query pipes: fail-closed (security)** — pre-defined SQL templates (Tinybird-style) with param binding + caching; `GET/POST /v1/pipes/{name}` sit outside `RequireAdmin`, so per-pipe `allowed_roles` is the *only* execute-path gate, via `policy.RoleAllowed`: exact allowlist membership (no `"*"`), admin always passes, empty/absent role and empty-string entries authorize nobody, and no `allowed_roles` → admin-only. Preserve and exercise via `testutil.RunRoleMatrix` / `StandardRoleMatrix` (see #159). Detail: architecture.md § `pipes/`. -14. **TypeScript SDK** — `@wavehouse/sdk`: typed query builder, real-time SSE over `fetch`, live queries (incrementable/decomposable/poll aggregation), codegen CLI. Exactly one runtime dependency — `eventsource-parser` (SSE framing, itself dependency-free); adding a second needs the same scrutiny the first got. The canonical client (see §SDK Sync). +14. **Client SDKs** — TypeScript (`@wavehouse/sdk` in `clients/ts/`) and Go (`github.com/Wave-RF/WaveHouse/clients/go`, package `wavehouse`, in `clients/go/`) are both canonical, officially supported clients with full API-tree parity. Each ships a typed query builder, real-time SSE over header-authenticated HTTP, live queries (incrementable/decomposable/poll aggregation). TypeScript adds a codegen CLI; Go row structs are hand-written. TypeScript has exactly one runtime dependency — `eventsource-parser` (SSE framing, itself dependency-free); adding a second needs the same scrutiny the first got. Go has zero third-party runtime dependencies (stdlib-only, hand-rolled SSE framing). See §SDK Sync. 15. **Observability invariants** — stdout always 100% (sampling is OTLP-push-only); WARN+ERROR always export at 100% (a non-configurable floor — don't expose it); gRPC OTel exporters dial lazily so an unreachable collector never blocks startup; the OTel Prometheus exporter uses a **private** `prometheus.Registry`. The OTLP endpoint/TLS/custom-CA/mTLS/headers are delegated to the OpenTelemetry SDK's standard `OTEL_EXPORTER_OTLP_*` env vars — `InitProvider` passes **no** endpoint/header options. Known gap, intentionally not patched in WaveHouse app code: the pinned gRPC logs exporter (`otlploggrpc` v0.19/v0.20) ignores the env TLS-cert vars, so a custom/private CA and mutual TLS apply to traces/metrics but **not** the logs signal (public-CA/system-roots TLS and plaintext still work for logs) — upstream bug open-telemetry/opentelemetry-go#6661. A malformed `OTEL_EXPORTER_OTLP_HEADERS` is logged and skipped by the SDK (fail-soft), not fatal. Preserve when touching the logger/sampler/provider. Detail: architecture.md § `observability/`. 16. **Bearer-token-only CORS posture (security)** — Bearer JWT on every request, no cookies/sessions; `corsMiddleware` deliberately **never** emits `Access-Control-Allow-Credentials` (not needed, and `*` + credentials is a spec violation browsers reject). `cors_allowed_origins` controls who can *read* responses, not cookie scope; CSRF protection is structural. Don't reintroduce cookie auth or `Allow-Credentials` without a design discussion — answers GitHub #29/#30. Code: `internal/api/router.go`. 17. **Non-fatal boot** — schema-discovery failure on boot is non-fatal: `cmd/wavehouse` records an `api.BootState`, binds `:8080`, serves 503 on `/livez`/`/readyz` with the diagnostic, and retries via `SchemaRegistry.RetryRefresh` (backoff 2s → 60s). Bounds supervisor restart loops. @@ -94,7 +94,7 @@ make fix # Auto-fix everything fixable (gofumpt, goimports, lint - make test # Go unit tests + coverage gate (alias for test-unit) make test-integration # Go integration tests + gate (Docker; testcontainers) make test-e2e # E2E SDK suite vs the cover binary + gate (Docker; testcontainers) -make test-ts # SDK vitest unit tests + coverage + gate +make test-sdk # Both SDK suites (Go + TS) + coverage + gates make ci # Full pre-push pipeline — run it the documented way (§Local-First Validation) make build # Compile → bin/wavehouse make dev # ClickHouse + hot-reload server on :8080 (Docker) @@ -125,7 +125,7 @@ Tooling notes (the non-obvious bits `make help` won't tell you): - **Policy helpers**: Use `policy.NewMemoryStore(p)` for in-memory policy testing without NATS. - **Pipes helpers**: Use `pipes.NewMemoryStore(queries...)` for in-memory pipes testing without NATS. - **Response assertions**: Use `testutil.AssertJSONResponse(t, rec, status, expected)` and `testutil.AssertJSONContains(t, rec, status, substring)`. -- **Coverage target**: 80% project-wide (CI enforces `threshold.total` in `.testcoverage.yml` against the merged unit + integration + e2e profile). Per-suite minima also enforced: unit 80%, integration 20%, e2e 60%, sdk 50%. Aim for 80%+ on new code. Coverage is published as a README badge (Go merged-total) and as PR comments via GitHub Code Quality — see `.github/workflows/README.md` "Coverage publishing"; the gate is unchanged. +- **Coverage target**: 80% project-wide (CI enforces `threshold.total` in `.testcoverage.yml` against the merged unit + integration + e2e profile). Per-suite minima also enforced: unit 80%, integration 20%, e2e 60%, go-sdk 75%, ts-unit 40%, ts-e2e 40%, ts-total 50%. The Go SDK (`clients/go`) is a nested module — invisible to the root module's `-coverpkg=./...`, so it is gated on its own `suites.go-sdk` floor and never merged into the project-wide total. Aim for 80%+ on new code. Coverage is published as a README badge (Go merged-total) and as PR comments via GitHub Code Quality — see `.github/workflows/README.md` "Coverage publishing"; the gate is unchanged. - **Every new function should have corresponding test cases.** Run `make lint` and `make test` before considering work complete. - **E2E tests via SDK**: The TypeScript SDK is the primary E2E test harness. Tests in `tests/e2e/sdk/` exercise the full pipeline (ingest → ClickHouse → query) and simultaneously validate backend behavior and SDK correctness. Use `make test-e2e` to run. Add new E2E scenarios as `tests/e2e/sdk/*.test.ts` files using helpers from `tests/e2e/sdk/helpers.ts`. - **Per-suite table isolation**: Each e2e test file owns its own ClickHouse tables — `clicks_` / `events_` / `users_`, generated from `tests/e2e/sdk/tables.ts` and created by `setup.ts`. A new test file must (1) add its suite name to `SUITES` in `tables.ts` and (2) get its names via `const T = suiteTables("")`, then reference `T.clicks` etc. — never a bare `clicks`. This makes cross-file *data* contamination structurally impossible. Files still run **sequentially** (`vitest.config.ts` `maxWorkers: 1`): running them in parallel is blocked by shared *global policy* state (several files read-modify-write the single policy document; `streaming.test.ts` flips the global `default_role`), so policy-mutating tests snapshot the full policy and restore it. Dropping `maxWorkers: 1` is a deferred follow-up tracked in #214 (per-table policy storage; see `docs/src/content/docs/ingest-pipeline.md` § Deferred). @@ -140,7 +140,7 @@ Tooling notes (the non-obvious bits `make help` won't tell you): make ci # Full parity with CI: parallel verify + builds + unit/SDK tests, then integration + E2E + cov ``` -If `make ci` passes locally, your commit has crossed the same gates CI will run — the CI workflow (`.github/workflows/ci.yml`) is a job DAG over the *same Makefile targets* (`verify`, `build-docs`, `test-unit`/`test-ts`, `test-integration`, `test-e2e`, `cov`), just spread across parallel runners. For workflow-only changes, read the YAML diff carefully and run `actionlint` if you have it installed. +If `make ci` passes locally, your commit has crossed the same gates CI will run — the CI workflow (`.github/workflows/ci.yml`) is a job DAG over the *same Makefile targets* (`verify`, `build-docs`, `test-unit`/`test-sdk`, `test-integration`, `test-e2e`, `cov`), just spread across parallel runners. For workflow-only changes, read the YAML diff carefully and run `actionlint` if you have it installed. ### Running `make ci` (for agents) @@ -294,7 +294,7 @@ Then run the reviewers relevant to the PR's diff (the same set from `scripts/pre Documentation *prose* — accuracy against the code, runnable examples, clarity, completeness — **and code↔docs sync** (code that changed but whose docs didn't) are reviewed by the **`docs-reviewer`** subagent, not the code-focused `pre-push-reviewer`. The canonical rubric is `.github/prompts/docs-review.md`. It complements the deterministic prose tools — misspell, markdownlint, starlight-links-validator — reviewing only what they can't, and it never edits docs or posts PR comments. -**Scope** is the canonical docs-prose set from `scripts/docs-prose.sh` — a *denylist*: every tracked `.md`/`.mdx` EXCEPT `.claude/**`, `.github/**`, `CHANGELOG.md`, `AGENTS.md`, `CLAUDE.md`, `*.draft.md`/`*.old.md`, `PERF-CLAIMS-REVIEW.md`, `docs/posthog-setup-report.md`. So it covers the Starlight site under `docs/src/content/` **and** the governance docs (`README.md`, the SDK readme `clients/ts/README.md`, `CONTRIBUTING.md`, `SECURITY.md`, `CODE_OF_CONDUCT.md`, `SUPPORT.md`) — new docs are picked up automatically. `CODE_OF_CONDUCT.md`/`SUPPORT.md` are deep-reviewed only on change or material suspicion. +**Scope** is the canonical docs-prose set from `scripts/docs-prose.sh` — a *denylist*: every tracked `.md`/`.mdx` EXCEPT `.claude/**`, `.github/**`, `CHANGELOG.md`, `AGENTS.md`, `CLAUDE.md`, `*.draft.md`/`*.old.md`, `PERF-CLAIMS-REVIEW.md`, `docs/posthog-setup-report.md`. So it covers the Starlight site under `docs/src/content/` **and** the governance docs (`README.md`, the SDK readmes `clients/ts/README.md` / `clients/go/README.md`, `CONTRIBUTING.md`, `SECURITY.md`, `CODE_OF_CONDUCT.md`, `SUPPORT.md`) — new docs are picked up automatically. `CODE_OF_CONDUCT.md`/`SUPPORT.md` are deep-reviewed only on change or material suspicion. **It is a hard pre-push gate**, run in parallel with the other pre-push reviewers (see §Pre-push self-review). Invoked with the **default (branch) scope** it emits a `VERDICT:` line; on `ship_it` the `review-marker.sh` SubagentStop hook writes `tmp/docs-reviewer-passed-`, which the push gate requires — unconditionally, on every PR-branch push (even code-only ones). Run it via **`/docs-review`**; with **no arg** that's the gating review (branch scope), while an explicit **path/glob** or **`all`** is **advisory** (no `VERDICT:`, no marker) for ad-hoc audits. The whole dev team runs Claude Code and this command is tracked in-repo, so everyone runs it themselves; there is intentionally **no PR/cloud path** for docs review. @@ -360,22 +360,32 @@ Diagrams render inside the Starlight content column (~46–58rem wide) as build- ## SDK Sync -The TypeScript SDK (`@wavehouse/sdk` in `clients/ts/`) is the canonical client and ships from this repo. When backend changes alter the public API surface, the SDK needs corresponding updates. The `pre-commit` git hook flags likely misses informationally; consult this table when deciding what to update. +The TypeScript SDK (`@wavehouse/sdk` in `clients/ts/`) and Go SDK (`github.com/Wave-RF/WaveHouse/clients/go`, in `clients/go/`) are both canonical, officially supported clients. Both ship from this repo with full API-tree parity. When backend changes alter the public API surface, both SDKs need corresponding updates. The `pre-commit` git hook flags likely misses informationally; consult this table when deciding what to update. | Backend change | SDK considerations | | -------------- | ------------------ | -| New user-facing API endpoint | Add a typed client method (in `clients/ts/src/client.ts` or the relevant subsystem file: `query-builder.ts`, `pipes.ts`, `policy.ts`, `stream/`, etc.); update the matching SDK doc page under `docs/src/content/docs/sdk/` (`queries`, `streaming`, `pipes`, `admin`, or `reference` by topic — plus the API tree in `reference.md`) | -| Change to JWT auth / role extraction | Update auth handling in `clients/ts/src/http.ts` and types in `clients/ts/src/client.ts` | -| Change to `EventMessage` / ingest event format | Update payload types in `clients/ts/src/` (some are codegen-regenerated — re-run the SDK codegen CLI) | -| New / changed structured query AST | Update `clients/ts/src/query-builder.ts` types + builder methods | -| Change to live-query aggregation classification | Update live-query helpers in `clients/ts/src/stream/` | -| Named pipes API change | Update `clients/ts/src/pipes.ts` | -| Policy / access-control change | Update `clients/ts/src/policy.ts` | -| ClickHouse schema-driven type changes | Re-run the SDK codegen CLI; commit regenerated types | +| New user-facing API endpoint | Add a typed client method in **both** SDKs (TS: `clients/ts/src/` — `client.ts`, `query-builder.ts`, `pipes.ts`, `policy.ts`, `stream/`; Go: `clients/go/` — corresponding file). Update the shared topic page under `docs/src/content/docs/sdk/` — add or extend the `` block so BOTH languages are covered on the same page (never a per-language page tree; see §SDK docs layout). Add a wire case to `clients/go/testdata/wire_cases.json` with dispatch in both conformance runners. | +| Change to JWT auth / role extraction | TS: `clients/ts/src/http.ts` + `client.ts`. Go: `clients/go/http.go` + `wavehouse.go`. | +| Change to `EventMessage` / ingest event format | Update payload types in both SDKs (re-run the TS codegen CLI where types are generated; Go row structs are hand-written). | +| New / changed structured query AST | TS: `clients/ts/src/query-builder.ts`. Go: `clients/go/query_builder.go` + `types.go`. | +| Change to live-query aggregation classification | TS: `clients/ts/src/stream/`. Go: `clients/go/live_query.go`. | +| Named pipes API change | TS: `clients/ts/src/pipes.ts`. Go: `clients/go/pipes.go`. | +| Policy / access-control change | TS: `clients/ts/src/policy.ts`. Go: `clients/go/policy.go`. | +| ClickHouse schema-driven type changes | Re-run the TS SDK codegen CLI and commit regenerated types; Go row structs are hand-written by the consumer. | Internal-only backend changes (middleware refactors, observability internals, dedup implementation, sweeper logic, NATS plumbing) generally don't need SDK updates. Use judgement — table above is the source of truth; nothing automated nudges you. -**The decision test**: would a `@wavehouse/sdk` user's *code* need to change to take advantage of (or be compatible with) this change? If yes, SDK update needed. If no (purely internal optimization), no. +**The decision test**: would a user's *code* need to change to take advantage of (or be compatible with) this change? If yes, both SDKs need updates. If no (purely internal optimization), no. + +### SDK docs layout + +The SDK docs are **topic-first, not language-first** — the decision from PR #313, carried into the Go SDK in PR #434. Adding a third language must not change the shape: + +- **One page per topic, shared by every language**: `/sdk/queries`, `/sdk/streaming`, `/sdk/pipes`, `/sdk/admin`, `/sdk/reference`. Prose that holds for every client stays outside the tabs; the language-specific code and caveats go inside a `` block, one `` per language. The `syncKey` is `lang` everywhere, so a reader picks a language once and it follows them across the whole tree. +- **One setup/caveats page per language, under `/sdk/setup/`**: `/sdk/setup/typescript`, `/sdk/setup/go`. Installation, client construction, auth, and the error model are genuinely per-language and live here. `/sdk/setup` itself is the hub — a table of every client (package, install command, version floor) plus cards to each language's page. +- **`/sdk` is the language-neutral overview.** No language sits at the root of `/sdk` — that was the trap the Go SDK's first draft fell into, leaving TypeScript un-prefixed and undocumented as a language. +- **Adding a language** = one new `/sdk/setup/` page + a row in the `/sdk/setup` table + one new `` per topic page + one entry in the sidebar's nested `Setup` group (which mirrors the `/sdk/setup/` route segment). It is never a parallel `/sdk//` tree, because that churns the topic URLs and doubles the pages to keep in sync. +- Because tabs need MDX, the topic pages are `.mdx` — remember §Markdown authoring rules: `make fix` does not auto-fix MDX, so unwrap WH001 wrapping by hand and keep a blank line between a JSX tag and a code fence. ## Common Tasks @@ -410,12 +420,14 @@ Internal-only backend changes (middleware refactors, observability internals, de 4. Use `testutil.MakeJWT(t, claims)` for auth tests, `testutil.NewTestSchemaRegistry(t, ...)` for schema-aware tests, `policy.NewMemoryStore(p)` for policy tests, `pipes.NewMemoryStore(queries...)` for pipes tests. 5. Use `testutil.AssertJSONResponse` and `testutil.AssertJSONContains` for HTTP handler assertions. 6. Run `make test` — it gates the unit-test coverage threshold from `.testcoverage.yml`, so a passing run already confirms coverage. -7. Aim for 80%+ coverage on new code. The project-wide CI-enforced minimum is 80% (merged unit + integration + e2e via `.testcoverage.yml`'s `threshold.total`); per-suite minima are unit 80%, integration 20%, e2e 60%, sdk 50%. +7. Aim for 80%+ coverage on new code. The project-wide CI-enforced minimum is 80% (merged unit + integration + e2e via `.testcoverage.yml`'s `threshold.total`); per-suite minima are unit 80%, integration 20%, e2e 60%, go-sdk 75%, ts-unit 40%, ts-e2e 40%, ts-total 50%. ## File Structure ```text cmd/ → Binary entry points (thin — just wiring) +clients/ts/ → TypeScript SDK (@wavehouse/sdk) +clients/go/ → Go SDK (github.com/Wave-RF/WaveHouse/clients/go) — nested module; file layout mirrors clients/ts/, plus testdata/wire_cases.json internal/api/ → HTTP layer (handlers, router, middleware, schema/DLQ/policy/pipes endpoints) internal/auth/ → JWT/JWKS authentication middleware (HMAC or JWKS, role extraction from claims) internal/cache/ → Caching (interface + L1/L2/tiered implementations) diff --git a/CHANGELOG.md b/CHANGELOG.md index 713b72c7..b19ac533 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,11 +10,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Added +- **Go SDK — official Go client with full API-tree parity against the TypeScript SDK** (`clients/go/` (new nested Go module), `tests/conformance/conformance_ts.mjs` (new), `docs/src/content/docs/sdk/setup/go.md` (new), `docs/src/content/docs/sdk/setup/typescript.mdx` (was `sdk/index.mdx`), `docs/src/content/docs/sdk/{index,queries,streaming,pipes,admin,reference}.mdx`, `docs/src/config/sidebar.ts`, `docs/src/components/Footer.astro`, `Makefile`, `.testcoverage.yml`, `scripts/cov/main.go`, `.github/workflows/ci.yml`, `.claude/commands/cover.md`, `AGENTS.md`, `CONTRIBUTING.md`, `README.md`, `SUPPORT.md`, `docs/src/content/docs/{development,architecture,getting-started,why-wavehouse,404,api,claude-code,deployment}.md`, `docs/src/content/docs/{access-control,pipes,reverse-proxy,index}.mdx`): install with `go get github.com/Wave-RF/WaveHouse/clients/go` (package `wavehouse`; a *nested* Go module with its own `go.mod`, invisible to root `go list` — hence the dedicated make targets). Zero third-party runtime dependencies, `context.Context`-first, generics for typed rows (`FetchTyped[Row]`, `Fetch[Row]`, `SQL[Row]`), an immutable chainable query builder with keyset pagination, real-time SSE streaming with reconnect/backfill (`Stream`, `LiveQuery`), admin namespaces (Schema/Policy/Pipes/DLQ/Sys), per-client `Headers` applied to REST and SSE alike (the Go analog of the TypeScript SDK's `options.headers`, and how an operator sends `X-Operator-Key`). A request carrying a credential — a bearer token or a configured header — never follows a redirect, on REST and SSE alike, because `net/http` strips only its own four sensitive headers across hosts and forwards configured ones verbatim; the refusal surfaces as `REDIRECT` (REST) or `SSE_REDIRECT` (stream). `MaxRetries` is a `*int` so that options constructed for `Headers` alone keep the default of 2 instead of reading Go's zero value as "no retries"; `wavehouse.Ptr` supplies the pointer. Wire-format parity is enforced by a shared fixture (`clients/go/testdata/wire_cases.json`) replayed by two conformance runners — Go (`conformance_test.go`) and TS (`tests/conformance/conformance_ts.mjs`) — each riding its own language's SDK target, both run by CI's unit job and local `make ci`. **Make targets follow one SDK family**: `test-sdk` runs both suites, `test-sdk-go` / `test-sdk-ts` run one (`test-ts` is renamed to the latter), and `test-sdk-go-e2e` drives a live server (`WAVEHOUSE_URL`/`WAVEHOUSE_AUTH`); the Go suites use gotestsum and honor `ARGS`/`V=1` like every other Go suite, while `test-sdk-ts` runs vitest. `make test` stays the `test-unit` alias it has always been. The nested module needs no targets of its own for static checks: `fmt-go`, `lint-go`, `tidy`, and `fix-go` each span both modules (`vulncheck` is still root-only — [#437](https://github.com/Wave-RF/WaveHouse/issues/437)). **Docs are topic-first, not per-language** (the decision from [#313](https://github.com/Wave-RF/WaveHouse/pull/313)): `/sdk/queries`, `/sdk/streaming`, `/sdk/pipes`, `/sdk/admin`, and `/sdk/reference` each carry a `` block per language-specific section, so the topic URLs never churn as SDKs are added, and each language keeps a setup/caveats page under `/sdk/setup/` — `/sdk/setup/typescript` (moved off the root `/sdk`, which is now a language-neutral overview) and `/sdk/setup/go`, indexed by `/sdk/setup`. Releases ride the tag-driven scheme already in place: `make release-sdk-go` cuts a `clients/go/vX.Y.Z` tag (`scripts/release.sh`), which the Go module proxy serves directly — no publish workflow needed. - **"Was this page helpful?" feedback widget on every docs page** (`docs/src/components/PageFeedback.astro` (new), `docs/src/components/Footer.astro`): a thumbs-up / thumbs-down vote below the page content, captured to PostHog as `docs_feedback` with `{ helpful, page }`. It renders from `Footer.astro`'s sidebar branch — the same indirection the Cloud CTA uses — rather than a per-page import or frontmatter flag, so every content page gets it automatically, including ones not written yet; it sits *below* the Cloud CTA on the pages that carry one, and splash pages (the homepage and 404) take the other footer branch and never render it. One vote per page per visitor: the choice is remembered in `localStorage` keyed by pathname, and a revisit renders the thanks message instead of re-prompting (storage is a nicety, not the record — a browser with storage disabled still votes). - **Settings-directory validation — `wavehouse validate [dir]`** (`internal/settings/` (new: `settings.go`, `validate.go`, `decode.go`, `finding.go`, + tests), `cmd/wavehouse/validate.go` (new, + tests), `cmd/wavehouse/main.go`): first piece of the file-based control plane (settings live in a directory of JSON documents — `roles.json`, `policies.json`, `pipes.json`, `config.json` — that a running instance will hot-reload; this change is validation-only — boot loading and reload wiring land separately). `settings.Validate(dir)` is the single gate every consumer of the directory runs: deliberately pure (no network, no ClickHouse — table/column existence stays with schema discovery, per Bring-Your-Own-Schema), and it collects **all** findings in one pass instead of failing on the first. Checks, layered: the directory holds exactly the four files (a missing file is an error — an empty document is `{}`, so absence always means deletion or a wrong path; any unexpected entry — file or directory — is an error so a typoed `polices.json` or a stray backup can't be silently ignored; dot-prefixed entries are the one carve-out, since erroring on vim swap files or the `..data` machinery Kubernetes ConfigMap mounts publish through would break hand editing and the cloud fan-out's mount pattern alike); strict JSON syntax (unknown fields rejected — the JSON form of the retired-config-key trap; empty/truncated files rejected, never read as an empty document; a leading UTF-8 byte order mark named as such instead of surfacing as a cryptic invalid-character error; a directory, unreadable file, or non-regular file (a FIFO would hang the read forever waiting for a writer; a stat gate rejects it — following symlinks, so Kubernetes ConfigMap mounts' symlink layout still passes) squatting on a settings filename named as the one real problem, not double-reported as "missing"; a top-level `null` rejected — the one well-formed document that decodes into a zero value without error, so it would silently read as "no settings"; trailing content rejected; duplicated object keys detected by a token-level pass, since `encoding/json` silently keeps the last copy); per-file shape rules (role names non-empty/unique, pipe names/SQL/param types, `config.json` bounds mirroring boot-config validation — its sections are the *tenant-owned* behavioral tunables (dedupe id_field/require_id plus per-table overrides under `dedupe.tables` — each entry overrides only the fields it names, resolving table → global → compiled default per field, so the effective id_field can never be empty — an explicit empty, whitespace-only, or whitespace-padded id_field is rejected at both levels, since an exact-match JSON key lookup would silently miss every row ([#222](https://github.com/Wave-RF/WaveHouse/issues/222)'s shape, unblocked by the file design since table names are runtime-resolved like policy grants); query default_max_rows, schema refresh_interval, CORS origins); platform-owned knobs like the SSE keepalives deliberately stay boot config); and cross-file referential integrity (every role a policy grant, `default_role`/`admin_role`, or pipe allowlist references must be declared in `roles.json`; an empty role string in a grant or allowlist is named as such — it matches no request and authorizes nobody). Warnings don't invalidate: a grant scoping the admin role (an unconditional bypass — dead config), `default_role` = admin, and a `default` on a required pipe parameter are flagged but legal. An empty `policies.json` means no policy — fail closed, matching deleted-policy semantics — and draws a warning naming the total lockout, so it announces itself at validation time instead of one 403 at a time. The CLI (`cmd/wavehouse/validate.go`, following the `health` subcommand pattern) takes the directory as an argument or from `WH_SETTINGS_DIR`, prints findings, and exits 0/1/2 (valid/invalid/usage) so CI and operators can gate config changes before they reach a running instance. The dispatch in `main.go` also grows `help` and `version` subcommands, and an unknown command is now a usage error instead of silently falling through and starting the server (`wavehouse validat` booting a listener is not a typo anyone wants); each subcommand parses its arguments with a stdlib `flag.FlagSet`, so `wavehouse -h` prints command-specific help and a stray flag or argument is a usage error rather than being silently swallowed. `WH_SETTINGS_DIR` has a single authority: `config.EnvSettingsDir`, with a reflection test pinning the `settings.dir` struct tag to it. The directory's location joins boot config as `settings.dir` (`WH_SETTINGS_DIR`; `internal/config/config.go`, `config.yaml`, `docs/src/content/docs/configuration.mdx`) — boot-tier by necessity, since it's the pointer the reload machinery follows; no default, same silent-misconfiguration reasoning as `policy.file_path`. - **Docs-site analytics for search, code copies, 404s, docs section, and live-demo connectivity** (`docs/src/components/DocsTracking.astro` (new), `docs/src/components/{PostHog,Footer,LiveDemo}.astro`): the site tracked its own CTAs but nothing a reader did on the way to one, so the questions that decide what to write next — what people search for and *don't* find, which snippets get copied, which dead links keep getting followed — had no data behind them. `docs_search` fires a second after the query settles rather than once per keystroke, carrying `query` and `result_count` read off Pagefind's own results message (the rendered list is capped at its page size, so counting the DOM would under-report); `result_count: 0` is the event worth having. `code_copied` (`page`, `language`) watches Expressive Code's copy buttons from the document rather than re-binding every code block on every navigation — the hero's install chip is not an EC block and keeps its own `hero_install_copied`. `docs_404` (`path`, `referrer`) turns broken inbound links into a list instead of a hunch. A `doc_section` property (the first path segment, `home` for `/`) puts every event in a docs area without each tracker carrying its own copy; it's stamped at capture time by a `before_send` hook in `posthog.init()` rather than `register()`, because a queued `register()` replays only after init has already captured the first hard-load `$pageview` — which would then carry the previous visit's persisted value — and `history_change` navigations update the URL before capture fires, so reading `location` in the hook is always current. `live_demo_connected` fires once per mount when the hero's SSE feed comes up rather than on its first row — named for what it measures (the demo backend answered), since a quiet minute on the repo is not a disengaged reader. The three site-wide trackers share one new `DocsTracking.astro` rendered from the footer (like `MermaidZoom` / `ScrollHints`) and delegate from `document`, since Pagefind, Expressive Code, and the 404 route all own their own markup — some of it created after page load. +### Fixed + +- **Go SDK client-side stream filters compare timestamps as instants, not as text** (`clients/go/stream.go`, `clients/go/stream_test.go`, `docs/src/content/docs/sdk/streaming.mdx`): the server canonicalizes every top-level `DateTime`/`DateTime64` value to RFC 3339 UTC before publishing (#402), so a payload reads `2026-06-21T04:00:00Z` while a caller's filter constant may name the same instant as `2026-06-21T06:00:00+02:00`. Compared as text those disagree in both directions — lexically the payload sorts *below* the constant, so `OpGte` withheld a row that was chronologically equal. Both sides now parse as instants, mirroring the server's row-filter rule for DateTime columns. Only unambiguous spellings count (RFC 3339 with an explicit offset or `Z`): a zone-less constant names an instant only relative to the column's declared timezone, which a stream subscriber doesn't have, so reading it as UTC would move the instant. Ordering an instant against a non-instant now fails closed rather than falling back to text comparison. Also fixes a missing column matching the literal string `""` through the equality fallback. The TypeScript SDK's `matchesFilters` has the same text-comparison behavior and needs the same change for parity. + +- **TypeScript SDK documentation corrections found while writing the Go SDK's parity docs** (`docs/src/content/docs/sdk/{reference,queries,streaming}.md`): the error-code table described `401` as "missing or invalid JWT" when a *missing* token is actually evaluated as `default_role` — succeeding or denied with `403`, never `401` (`internal/auth/auth.go`, `internal/api/errors.go`) — and only a present-but-invalid or expired token yields `401`. `.aggregate()` was documented as accepting a "custom fn", but the server enforces an allowlist (`internal/query/builder.go`); the allowed set is now listed. Live queries gained a caution for a real footgun: the backfill dedup boundary comes from the fetched rows' `received_timestamp`, so a `.select(...)` projection omitting that column silently disables dedup and delivers overlap-window events twice. No SDK code changed — these were pre-existing gaps between the TS docs and the server's behavior. + ## [0.1.0] - 2026-08-19 The first public release. Everything below shipped in it — the sections are grouped the way Keep a Changelog asks for, but since there is no previous release to compare against, a reader upgrading from nothing can treat the whole file as "Added". The date is the intended cut date; correct it if tagging slips, and move anything merged in between up from `## Unreleased`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a1d656b8..3ce264fd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,13 +39,14 @@ Open a [feature request issue](https://github.com/Wave-RF/WaveHouse/issues/new?t The pre-push hook (installed by `make tools`) blocks a push until the tree has been validated locally: a code change needs `make ci`, a docs/prose-only change needs only `make verify` (the same split CI makes). `make lint` / `make test` / `make build` are fast inner-loop subsets. -2. Write tests for new functionality. Unit tests go alongside the code in `internal/`. Integration tests go in `tests/` with the `//go:build integration` tag. +2. Write tests for new functionality. Unit tests go alongside the code in `internal/`; SDK tests live in `clients/ts/src/` and `clients/go/`. Integration tests go in `tests/` with the `//go:build integration` tag. 3. Update documentation if your change affects: - API endpoints → update `docs/src/content/docs/api.md` - Configuration options → update `docs/src/content/docs/configuration.mdx` - Deployment → update `docs/src/content/docs/deployment.md` - Architecture → update `docs/src/content/docs/architecture.md` + - Client SDK surface → update **both** SDKs (`clients/ts/src/`, `clients/go/`), the shared topic pages under `docs/src/content/docs/sdk/` (a `` block per language-specific section, with one `` per language) plus the per-language setup pages `docs/src/content/docs/sdk/setup/typescript.mdx` / `docs/src/content/docs/sdk/setup/go.md`, and the shared wire fixture `clients/go/testdata/wire_cases.json`; see AGENTS.md §SDK Sync 4. Follow the commit message format (see below). @@ -89,7 +90,7 @@ test(cache): add tiered cache stampede test ## Code Style -- **Formatting**: Code must be formatted with `gofumpt` (a strict superset of `gofmt`). `make fmt` checks it (CI runs the same target); `make fix` applies it. +- **Formatting**: Code must be formatted with `gofumpt` (a strict superset of `gofmt`). `make fmt` checks both modules — the root one and the nested `clients/go` — as do `make lint` and `make tidy`. `make fix` applies the corresponding fixes to both. - **Linting**: All lint checks in `.golangci.yml` must pass (see `make lint`). - **Naming**: Follow [Go naming conventions](https://go.dev/doc/effective_go#names). - **Interfaces**: Define interfaces where they are consumed, not where they are implemented. diff --git a/Makefile b/Makefile index 3920763c..89cf29f7 100644 --- a/Makefile +++ b/Makefile @@ -192,13 +192,15 @@ ACTIONLINT_VERSION := v1.7.12 ACTIONLINT := $(LOCAL_BIN)/actionlint-$(ACTIONLINT_VERSION) # --- Coverage Directories ----------------------------------------------------- -# One path per suite. Internal layout (managed by scripts/coverage.sh): +# One path per suite. Internal layout (managed by scripts/cov): # $(COV_X)/data/ binary covdata (covmeta.* / covcounters.*) # $(COV_X)/coverage.txt rendered textfmt profile # $(COV_X)/coverage.html rendered HTML report COV_UNIT := tmp/coverage/unit COV_INT := tmp/coverage/integration COV_E2E := tmp/coverage/e2e +# Nested module at clients/go — same layout, own gate, outside COV_TOTAL. +COV_GOSDK := tmp/coverage/go-sdk COV_TOTAL := tmp/coverage/total # --- Coverage Thresholds ------------------------------------------------------ @@ -330,6 +332,12 @@ obs-front: ## Start local OTel Front UI # Dynamically find all directories containing Go files, safely ignoring hidden folders like .worktrees GO_DIRS := $(shell go list -f '{{.Dir}}' ./...) +# The Go SDK is a NESTED module (its own go.mod), so it is invisible to the +# root `go list ./...` above — every Go check names it explicitly. Path-based +# tools (gofumpt, goimports) take it as an extra argument; module-scoped tools +# (go mod tidy, golangci-lint) need a second invocation from inside it. +GO_SDK_DIR := clients/go + # fmt / lint / fix: one Biome binary (biome.json) scans the whole workspace # (SDK + e2e + docs); Markdown is owned separately by markdownlint-cli2 (rules in # .markdownlint.json, globs in .markdownlint-cli2.jsonc) — Biome only does @@ -357,18 +365,25 @@ fmt: fmt-go fmt-ts ## Check formatting across Go (gofumpt) + TS (Biome). Run `ma .PHONY: fmt-go fmt-go: - $(call run,gofumpt (Go fmt),$(GOFUMPT) -l $(GO_DIRS) | (! grep .),run make fix to apply formatting) + $(call run,gofumpt (Go fmt),$(GOFUMPT) -l $(GO_DIRS) $(GO_SDK_DIR) | (! grep .),run make fix to apply formatting) .PHONY: fmt-ts fmt-ts: pnpm-install $(call run,Biome (format),$(PNPM) -s -w run format,run make fix to apply formatting) .PHONY: lint -lint: lint-go lint-ts lint-md lint-prose ## Lint across Go (golangci-lint) + TS/JSON (Biome) + Markdown (markdownlint) + docs prose (misspell). Run `make fix` to apply --fix. - +lint: lint-go lint-ts lint-md lint-prose ## Lint across Go (golangci-lint, both modules) + TS/JSON (Biome) + Markdown (markdownlint) + docs prose (misspell). Run `make fix` to apply --fix. + +# lint-go spans BOTH Go modules, the same way fmt-go does — one entry point per +# tool, not one per module. It takes two invocations rather than two targets +# because golangci-lint is module-scoped: `run ./...` resolves packages through +# the go.mod in its working directory, so the nested SDK needs its own run from +# inside $(GO_SDK_DIR). It picks up this repo's .golangci.yml either way +# (golangci-lint walks up to find the config). .PHONY: lint-go lint-go: $(GOLANGCI_LINT) go-mod-download $(call run,golangci-lint,$(GOLANGCI_LINT) run ./... --allow-parallel-runners,run make fix to auto-fix what is fixable) + $(call run,golangci-lint (Go SDK),cd $(GO_SDK_DIR) && $(GOLANGCI_LINT) run ./... --allow-parallel-runners,run make fix to auto-fix what is fixable) .PHONY: lint-ts lint-ts: pnpm-install @@ -447,9 +462,12 @@ endif # tidy: read-only check via `go mod tidy -diff` (Go 1.23+). Prints the # unified diff that would be applied and exits non-zero if anything is off, # without touching go.mod / go.sum. Safe to run in parallel with fmt/lint. +# Spans both modules (see $(GO_SDK_DIR)) so `make verify` checks exactly what +# `make fix` would rewrite — the nested go.mod included. .PHONY: tidy -tidy: ## Verify go.mod/go.sum are tidy (run `make fix` to apply) +tidy: ## Verify go.mod/go.sum are tidy in both modules (run `make fix` to apply) $(call run,go.mod tidy,go mod tidy -diff,run make fix to tidy go.mod and go.sum) + $(call run,go.mod tidy (Go SDK),cd $(GO_SDK_DIR) && go mod tidy -diff,run make fix to tidy go.mod and go.sum) # fix: apply auto-fixes everywhere, fanned out into three tracks that touch # disjoint files — Go (.go + go.mod/sum), TS/JS/JSON (Biome), Markdown — so they @@ -473,13 +491,19 @@ fix-docs: pnpm-install @$(MAKE) fix-md @$(MAKE) fix-prose +# The gofumpt/goimports passes take $(GO_SDK_DIR) as an extra path argument; +# `go mod tidy` and `golangci-lint` are module-scoped, so the nested SDK gets +# its own `cd $(GO_SDK_DIR) &&` line for each. That `cd` is the only difference +# between the two golangci-lint lines below — they are not a duplicate. .PHONY: fix-go fix-go: $(GOLANGCI_LINT) - @echo "$(CYAN)==> Applying Go auto-fixes (tidy + gofumpt + goimports + lint --fix)...$(RESET)" + @echo "$(CYAN)==> Applying Go auto-fixes (tidy + gofumpt + goimports + lint --fix, both modules)...$(RESET)" @go mod tidy - @$(GOFUMPT) -w $(GO_DIRS) - @$(GOIMPORTS) -w $(GO_DIRS) + @cd $(GO_SDK_DIR) && go mod tidy + @$(GOFUMPT) -w $(GO_DIRS) $(GO_SDK_DIR) + @$(GOIMPORTS) -w $(GO_DIRS) $(GO_SDK_DIR) @$(GOLANGCI_LINT) run --fix ./... --allow-parallel-runners + @cd $(GO_SDK_DIR) && $(GOLANGCI_LINT) run --fix ./... --allow-parallel-runners .PHONY: fix-ts fix-ts: pnpm-install @@ -525,9 +549,11 @@ fix-prose: $(MISSPELL) # slowest tool, not the slowest *group* (e.g. golangci no longer drags Biome + # markdownlint along behind it). # -# Leaves (14): tidy, fmt-go (gofumpt), lint-go (golangci), vulncheck on the Go -# side; lint-ts (biome check) + lint-md (markdownlint) + lint-prose (misspell, -# docs spelling) + test-md-rules (node --test over the WH001/WH002 fixtures) +# Leaves (14): tidy, fmt-go (gofumpt), lint-go (golangci) and vulncheck on the +# Go side — the first three span BOTH modules (root + the nested clients/go +# SDK), so the SDK needs no leaves of its own; lint-ts (biome check) + lint-md +# (markdownlint) + lint-prose (misspell, docs spelling) + test-md-rules +# (node --test over the WH001/WH002 fixtures) # for JS/TS + Markdown + prose; lint-sh (shellcheck), lint-gha (actionlint), # test-classify-paths and test-release-channel for the tooling; # check-docs (astro check — the only leaf that writes, to docs/.astro/, and @@ -565,10 +591,18 @@ typecheck-ts: pnpm-install # go-mod-download is a no-doc intermediate target — every Go-toolchain target # (build/test/lint variants) declares it as a prereq so `make -j` doesn't # kick off N parallel `go mod download` calls racing on the module cache. -# Symmetric with pnpm-install for the Node side. +# Symmetric with pnpm-install for the Node side. It warms BOTH modules, so a +# target that reaches into $(GO_SDK_DIR) inherits the same guarantee from the +# one prereq (the SDK is stdlib-only today, so its half is ~free — it is there +# so the invariant holds if that ever changes). Only the SDK half swallows its +# output on success: with no requirements to fetch it prints a "no module +# dependencies to download" line that would otherwise show up ahead of every +# Go target. The root half stays unfiltered so a cold cache's "go: downloading" +# progress is still visible. .PHONY: go-mod-download go-mod-download: @go mod download + @cd $(GO_SDK_DIR) && out=$$(go mod download 2>&1) || { printf '%s\n' "$$out" >&2; exit 1; } .PHONY: $(BINARIES) $(BINARIES): go-mod-download @@ -657,7 +691,7 @@ branding-docs: ## Regenerate docs logo/favicon/OG assets from docs/src/assets/br # # Everything is driven directly via `pnpm --filter ` — no per-subproject # Makefiles. The user-facing Node targets live inline in their natural verb -# sections (build-ts/dev-ts/test-ts/clean-ts; build-docs/dev-docs/preview-docs/ +# sections (build-ts/dev-ts/clean-ts and test-sdk-ts; build-docs/dev-docs/preview-docs/ # branding-docs/clean-docs) and declare pnpm-install as a prereq so a fresh # clone or a changed lockfile is handled lazily. `make tools` does the full # bootstrap. @@ -742,7 +776,7 @@ test-unit: go-mod-download ## Run Go unit tests + render coverage + gate thresho @if [ -z "$(COV_DEFER)" ]; then go run ./scripts/cov render unit; fi # Hidden alias: `make test` matches `go test ./...` muscle memory; test-unit -# is the explicit form. +# is the explicit form. Server unit tests only — the SDK suites are `test-sdk`. .PHONY: test test: test-unit @@ -772,28 +806,72 @@ test-e2e: build-ts build-cover ## Run E2E SDK suite against cover binary + rende go run ./scripts/orchestrator @if [ -z "$(COV_DEFER)" ]; then go run ./scripts/cov render e2e; fi -# test-ts: vitest unit tests for the SDK, always with v8 coverage. Standalone -# it also gates against suites.ts-unit (via vitest's --coverage.thresholds); -# under COV_DEFER it only collects, leaving the gate to `make cov` (cov report) -# so CI emits one consolidated coverage block. THRESHOLD is read live from -# .testcoverage.yml via scripts/cov; override with -# `make test-ts ARGS='--coverage.thresholds.statements=70'`. -.PHONY: test-ts -test-ts: pnpm-install ## Run SDK vitest unit tests + coverage + gate against suites.ts-unit - @printf "$(CYAN)==> Running SDK unit tests...$(RESET)\n" +# --- SDK test suites ---------------------------------------------------------- +# One naming family for every SDK suite: `test-sdk` runs them all, `test-sdk-go` +# / `test-sdk-ts` run one language. Each language target also runs ITS HALF of +# the cross-language wire-format conformance suite (both halves replay +# clients/go/testdata/wire_cases.json) — the Go half is an ordinary test file in +# the SDK package, so folding the TS half into test-sdk-ts keeps the two +# symmetric and costs no extra target. +.PHONY: test-sdk +test-sdk: test-sdk-go test-sdk-ts ## Run both SDK test suites (Go + TypeScript) incl. cross-language wire conformance + +# test-sdk-go: unit tests for the nested clients/go module — outside test-unit's +# scope, so it needs its own target (-race: the SDK's streaming subsystem is the +# most concurrent code in the repo). gotestsum comes from the ROOT module's tool +# directives, which `go tool` can only resolve there — hence `go tool -n` to +# resolve the binary before cd-ing into the nested module. Covdata lands in the +# same layout as the root-module suites, so `cov render go-sdk` gates it with no +# new machinery, but it is never merged into the Go total — see the go-sdk +# comment in .testcoverage.yml. +.PHONY: test-sdk-go +test-sdk-go: go-mod-download ## Run Go SDK (clients/go, a nested module) unit + conformance tests + render coverage + gate threshold + @printf "$(CYAN)==> Running Go SDK tests...$(RESET)\n" + @rm -rf $(COV_GOSDK)/data && mkdir -p $(COV_GOSDK)/data + @gotestsum=$$(go tool -n gotestsum) && cd $(GO_SDK_DIR) && \ + GOCOVERDIR="$(CURDIR)/$(COV_GOSDK)/data" "$$gotestsum" --format $(GOTESTSUM_FMT) -- \ + -cover -coverpkg=./... -race -timeout 60s ./... $(ARGS) \ + -args -test.gocoverdir="$(CURDIR)/$(COV_GOSDK)/data" + @if [ -z "$(COV_DEFER)" ]; then go run ./scripts/cov render go-sdk; fi + +# test-sdk-ts: vitest unit tests for the TS SDK, always with v8 coverage, then +# the TS half of the wire conformance suite (which replays the fixture through +# the built dist — hence the build-ts prereq). Standalone it also gates against +# suites.ts-unit (via vitest's --coverage.thresholds); under COV_DEFER it only +# collects, leaving the gate to `make cov` (cov report) so CI emits one +# consolidated coverage block. THRESHOLD is read live from .testcoverage.yml via +# scripts/cov; override with +# `make test-sdk-ts ARGS='--coverage.thresholds.statements=70'`. +.PHONY: test-sdk-ts +test-sdk-ts: pnpm-install build-ts ## Run TS SDK vitest unit + conformance tests + coverage + gate against suites.ts-unit + @printf "$(CYAN)==> Running TypeScript SDK tests...$(RESET)\n" @rm -rf tmp/coverage/ts-unit && mkdir -p tmp/coverage/ts-unit @TS_UNIT_COVERAGE_DIR="$(CURDIR)/tmp/coverage/ts-unit" \ $(PNPM) --filter $(SDK_NAME) exec vitest run --coverage \ $(if $(COV_DEFER),,--coverage.thresholds.statements=$$(go run ./scripts/cov threshold ts-unit)) $(ARGS) + @printf "$(CYAN)==> Running TypeScript wire-format conformance...$(RESET)\n" + @node tests/conformance/conformance_ts.mjs @if [ -z "$(COV_DEFER)" ]; then printf "$(GREEN)==> ts-unit gate passed$(RESET) HTML: tmp/coverage/ts-unit/index.html\n"; fi +# test-sdk-go-e2e: the Go SDK against a LIVE server (WAVEHOUSE_URL + +# WAVEHOUSE_AUTH), behind the `e2e` build tag. Unlike test-e2e it brings up +# nothing itself and collects no coverage; wiring it into the orchestrator so it +# runs in CI with a coverage gate — the way the TS SDK's e2e half already does — +# is tracked in #518. +.PHONY: test-sdk-go-e2e +test-sdk-go-e2e: go-mod-download ## Run Go SDK E2E tests against a live WaveHouse instance (WAVEHOUSE_URL, WAVEHOUSE_AUTH) + @printf "$(CYAN)==> Running Go SDK E2E tests...$(RESET)\n" + @gotestsum=$$(go tool -n gotestsum) && cd $(GO_SDK_DIR) && \ + "$$gotestsum" --format $(GOTESTSUM_FMT) -- \ + -tags e2e -count=1 -timeout 60s ./... $(ARGS) + # Aggregator: recipe-based with $(MAKE) calls so suites run sequentially even # under `make -j N`. The suites bind ports / spin testcontainers / start the # release binary, so concurrent execution is unsafe. .PHONY: test-all test-all: ## Run all suites sequentially + one consolidated Go + TS coverage report + gates @$(MAKE) test-unit COV_DEFER=1 - @$(MAKE) test-ts COV_DEFER=1 + @$(MAKE) test-sdk COV_DEFER=1 @$(MAKE) test-integration COV_DEFER=1 @$(MAKE) test-e2e COV_DEFER=1 @$(MAKE) cov @@ -832,7 +910,7 @@ cov: go-mod-download ## Consolidated coverage report (Go + TS) + gate against th # marker that standalone `make verify` writes is instead written by ci's own # `ci-marker.sh write` below — it touches both the ci and verify markers. .PHONY: ci-parallel -ci-parallel: verify-parallel build build-cover build-ts build-docs test test-ts +ci-parallel: verify-parallel build build-cover build-ts build-docs test test-sdk .PHONY: ci ci: ## Full pipeline — parallel checks, then sequential heavy suites + coverage @@ -988,7 +1066,7 @@ clean-all: clean clean-test clean-tools ## Full reset — clean + clean-test + c # tools: bootstrap a fresh clone. # - Installs pinned external binaries to .bin/ (golangci-lint, air, misspell). # - Downloads Go modules so go.mod tool deps are available offline. -# - Installs SDK + E2E pnpm deps so test-ts / test-e2e are runnable +# - Installs SDK + E2E pnpm deps so test-sdk-ts / test-e2e are runnable # without a separate manual setup step. # # Note: go.mod tool deps (gotestsum, gofumpt, etc.) are *downloaded* by diff --git a/README.md b/README.md index 4a02098a..a54bab37 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ If you're building user-facing analytics, WaveHouse is like **Supabase for Click - **Query** — in-process Ristretto cache + `singleflight` coalescing; type-safe structured query AST; Tinybird-style named pipes (parameterized SQL endpoints). - **Real-time** — native SSE push, broadcast *before* the ClickHouse flush, with JetStream gap-fill for late/reconnecting clients. - **Security** — Hasura-style per-table, per-role column + row policies with JWT claim templating, stored in NATS KV. -- **Client** — `@wavehouse/sdk`: TypeScript client with query builder, live queries, streaming, and schema codegen; one runtime dependency (an SSE frame parser, ~1.4 KB gzipped). +- **Client SDKs** — TypeScript (`@wavehouse/sdk`) and Go (`github.com/Wave-RF/WaveHouse/clients/go`): query builder, live queries, streaming, and typed rows, over one shared wire format. ## How it compares diff --git a/biome.json b/biome.json index e29be7f3..700f3729 100644 --- a/biome.json +++ b/biome.json @@ -10,6 +10,7 @@ "clients/ts/src/**", "clients/ts/*.{ts,js,mjs,cjs,json}", "tests/e2e/sdk/**/*.{ts,js,mjs,cjs,json}", + "tests/conformance/**/*.{ts,js,mjs,cjs,json}", "docs/**/*.{ts,js,mjs,cjs,json}", "scripts/**/*.{ts,js,mjs,cjs}" ] diff --git a/clients/go/README.md b/clients/go/README.md new file mode 100644 index 00000000..f1b67448 --- /dev/null +++ b/clients/go/README.md @@ -0,0 +1,59 @@ +# WaveHouse Go SDK + +Official Go client for [WaveHouse](https://github.com/Wave-RF/WaveHouse) — a schema-aware real-time API gateway for ClickHouse. Zero third-party runtime dependencies, SSE parser included. + +**Full documentation: [wavehouse.dev/sdk/go](https://wavehouse.dev/sdk/setup/go)** (setup); the usage guides below cover both SDKs, tabbed by language. + +## Install + +```bash +go get github.com/Wave-RF/WaveHouse/clients/go +``` + +Requires Go 1.24 or newer. + +## Quick start + +```go +package main + +import ( + "context" + "fmt" + "log" + + wavehouse "github.com/Wave-RF/WaveHouse/clients/go" +) + +func main() { + wh := wavehouse.NewClient(wavehouse.Config{ + BaseURL: "http://localhost:8080", + Auth: wavehouse.StaticToken("your-jwt"), // omit for unauthenticated access + }) + + page, err := wh.From("clicks"). + Select("page", "button"). + Where("page", wavehouse.OpEq, "/home"). + Limit(10). + FetchUntyped(context.Background()) + if err != nil { + log.Fatal(err) + } + for _, row := range page.Data { + fmt.Println(row["page"], row["button"]) + } +} +``` + +## Documentation + +- [Go SDK setup](https://wavehouse.dev/sdk/setup/go) — client config, auth, typed rows via generics, error handling. +- [Queries](https://wavehouse.dev/sdk/queries) — tables, the query builder, inserts, pagination, raw SQL. +- [Streaming & Live Queries](https://wavehouse.dev/sdk/streaming) — SSE streams, client-side filtering, backfill-then-live. +- [Pipes](https://wavehouse.dev/sdk/pipes) — execute and manage named query pipes. +- [Admin & System](https://wavehouse.dev/sdk/admin) — schema, policy, DLQ stats, health. +- [Reference & CLI](https://wavehouse.dev/sdk/reference) — error codes, the full API tree, and the ClickHouse-to-Go field mapping for row structs. + +## License + +Apache-2.0 diff --git a/clients/go/client_test.go b/clients/go/client_test.go new file mode 100644 index 00000000..d2ad710f --- /dev/null +++ b/clients/go/client_test.go @@ -0,0 +1,120 @@ +package wavehouse + +import ( + "context" + "encoding/json" + "testing" +) + +func TestNewClient(t *testing.T) { + tests := []struct { + name string + cfg Config + wantBaseURL string + wantMaxRetries int + }{ + { + name: "defaults", + cfg: Config{BaseURL: "http://localhost:8080"}, + wantBaseURL: "http://localhost:8080", + wantMaxRetries: 2, + }, + { + name: "trailing slashes are stripped", + cfg: Config{BaseURL: "http://localhost:8080///"}, + wantBaseURL: "http://localhost:8080", + wantMaxRetries: 2, + }, + { + name: "MaxRetries is honored", + cfg: Config{BaseURL: "http://localhost:8080", Options: &ClientOptions{MaxRetries: Ptr(5)}}, + wantBaseURL: "http://localhost:8080", + wantMaxRetries: 5, + }, + { + // The regression a plain int caused: options set for an unrelated + // reason silently disabled retries, because 0 is both "unset" and + // "none". Headers is the field the docs push operators toward. + name: "options set only for Headers keep the default", + cfg: Config{BaseURL: "http://localhost:8080", Options: &ClientOptions{Headers: map[string]string{"X-Operator-Key": "k"}}}, + wantBaseURL: "http://localhost:8080", + wantMaxRetries: 2, + }, + { + name: "Ptr(0) disables retries", + cfg: Config{BaseURL: "http://localhost:8080", Options: &ClientOptions{MaxRetries: Ptr(0)}}, + wantBaseURL: "http://localhost:8080", + wantMaxRetries: 0, + }, + { + name: "a negative MaxRetries clamps to 0", + cfg: Config{BaseURL: "http://localhost:8080", Options: &ClientOptions{MaxRetries: Ptr(-3)}}, + wantBaseURL: "http://localhost:8080", + wantMaxRetries: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := NewClient(tt.cfg) + if c.ctx.baseURL != tt.wantBaseURL { + t.Errorf("want baseURL %q, got %q", tt.wantBaseURL, c.ctx.baseURL) + } + if c.ctx.maxRetries != tt.wantMaxRetries { + t.Errorf("want maxRetries %d, got %d", tt.wantMaxRetries, c.ctx.maxRetries) + } + }) + } +} + +func TestNewClient_HasNamespaces(t *testing.T) { + c := NewClient(Config{BaseURL: "http://localhost:8080"}) + // Compared as concrete typed pointers, not boxed into map[string]any: a nil + // typed pointer in an interface is never == nil, so the map form passed + // even if NewClient stopped assigning a namespace entirely. + if c.Sys == nil { + t.Error("Sys namespace is nil") + } + if c.Schema == nil { + t.Error("Schema namespace is nil") + } + if c.Policy == nil { + t.Error("Policy namespace is nil") + } + if c.Pipes == nil { + t.Error("Pipes namespace is nil") + } + if c.DLQ == nil { + t.Error("DLQ namespace is nil") + } +} + +func TestClient_SQL(t *testing.T) { + c, reqs := recordingClient(t, jsonResponse([]map[string]any{{"x": 1}})) + rows, err := SQL[map[string]any](context.Background(), c, "SELECT 1") + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 { + t.Fatalf("want 1 row, got %d", len(rows)) + } + got := <-reqs + if got.method != "POST" || got.path != "/v1/ops/query" { + t.Fatalf("want POST /v1/ops/query, got %s %s", got.method, got.path) + } + if body := string(got.body); body != `{"sql":"SELECT 1"}` { + t.Fatalf(`want {"sql":"SELECT 1"}, got %s`, body) + } +} + +func TestPolicyFilter_MarshalOperators(t *testing.T) { + empty := "" + raw, err := json.Marshal(PolicyFilter{Eq: &empty}) + if err != nil { + t.Fatal(err) + } + // Intentional empty-string comparison survives; unset operators are + // omitted entirely, never sent as null. + if string(raw) != `{"_eq":""}` { + t.Fatalf(`want {"_eq":""}, got %s`, raw) + } +} diff --git a/clients/go/conformance_test.go b/clients/go/conformance_test.go new file mode 100644 index 00000000..81496a59 --- /dev/null +++ b/clients/go/conformance_test.go @@ -0,0 +1,341 @@ +package wavehouse + +import ( + "context" + _ "embed" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strings" + "sync" + "testing" +) + +// wireCasesJSON embeds the shared wire-format fixture — the same file the TS +// runner replays (tests/conformance/conformance_ts.mjs) — so the test binary +// is self-contained: it works from a module archive or a standalone checkout +// without depending on paths outside the Go module. +// +//go:embed testdata/wire_cases.json +var wireCasesJSON []byte + +// wireCase is one entry in the shared wire_cases.json fixture. +type wireCase struct { + Name string `json:"name"` + Endpoint string `json:"endpoint"` + Table string `json:"table"` + Operations []wireOp `json:"operations"` + PipeName string `json:"pipe_name"` + PipeParams map[string]any `json:"pipe_params"` + PipeDefBody json.RawMessage `json:"pipe_def"` + PolicyBody json.RawMessage `json:"policy_body"` + SQL string `json:"sql"` + ExpectedPath string `json:"expected_path"` + ExpectedMethod string `json:"expected_method"` + ExpectedContentType string `json:"expected_content_type"` + ExpectedBody json.RawMessage `json:"expected_body"` + ExpectedRawBody *string `json:"expected_raw_body"` +} + +type wireOp struct { + Method string `json:"method"` + Args []any `json:"args"` +} + +// captured holds the HTTP request details from a single SDK call. +type captured struct { + method string + path string // path + query string + contentType string + body string +} + +// logErr surfaces SDK-call errors the assertions otherwise ignore: they only +// inspect the captured request, but a call that fails before sending should +// name the real cause. +func logErr(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Logf("SDK call returned error (request may still be valid): %v", err) + } +} + +// errOf drops the value of a two-result SDK call, keeping only the error. +func errOf[T any](_ T, err error) error { return err } + +// jsonArg decodes a fixture field into the SDK request type it stands for. +func jsonArg[T any](t *testing.T, field string, raw json.RawMessage) T { + t.Helper() + var v T + if err := json.Unmarshal(raw, &v); err != nil { + t.Fatalf("parse %s: %v", field, err) + } + return v +} + +func TestConformance_WireFormat(t *testing.T) { + var cases []wireCase + if err := json.Unmarshal(wireCasesJSON, &cases); err != nil { + t.Fatalf("parse wire_cases.json: %v", err) + } + + for _, tc := range cases { + t.Run(tc.Name, func(t *testing.T) { + // capt is written on the server goroutine and read on the test + // goroutine; the mutex is what makes that visible under -race. + var mu sync.Mutex + var capt captured + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + capt.method = r.Method + capt.path = r.URL.RequestURI() + capt.contentType = r.Header.Get("Content-Type") + raw, _ := io.ReadAll(r.Body) + capt.body = string(raw) + mu.Unlock() + + // Return valid JSON so the SDK doesn't error on decode. + w.Header().Set("Content-Type", "application/json") + switch { + case strings.HasPrefix(r.URL.Path, "/v1/ops/dlq"): + _ = json.NewEncoder(w).Encode(DLQStats{Tables: map[string]int{}, Total: 0}) + case strings.HasPrefix(r.URL.Path, "/v1/ops/schema") && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]TableSchema{}) + case r.URL.Path == "/v1/ops/policy/validate" && r.Method == "POST": + _ = json.NewEncoder(w).Encode(ValidationResult{Valid: true}) + case strings.HasPrefix(r.URL.Path, "/v1/ops/policy") && r.Method == "GET": + _ = json.NewEncoder(w).Encode(Policy{Tables: map[string]TablePolicy{}}) + case strings.HasPrefix(r.URL.Path, "/v1/ops/pipes/") && r.Method == "GET": + _ = json.NewEncoder(w).Encode(Pipe{Name: "test", SQL: "SELECT 1"}) + case r.URL.Path == "/v1/ops/pipes" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]Pipe{}) + case strings.HasPrefix(r.URL.Path, "/v1/ingest"): + if r.Header.Get("Content-Type") == "application/x-ndjson" { + _ = json.NewEncoder(w).Encode(InsertResult{}) + } else { + _ = json.NewEncoder(w).Encode(map[string]any{"ok": true}) + } + case r.URL.Path == "/v1/health": + _ = json.NewEncoder(w).Encode(map[string]any{"status": "ok"}) + default: + _ = json.NewEncoder(w).Encode([]map[string]any{}) + } + })) + defer srv.Close() + + c := NewClient(Config{ + BaseURL: srv.URL, + HTTPClient: srv.Client(), + Options: &ClientOptions{MaxRetries: Ptr(0)}, + }) + ctx := context.Background() + + switch tc.Endpoint { + case "query": + logErr(t, errOf(applyOps(t, tc.Table, c, tc.Operations).FetchUntyped(ctx))) + case "ingest", "ingest_batch": + logErr(t, errOf(c.From(tc.Table).Insert(ctx, insertArg(t, tc)))) + case "pipe": + logErr(t, errOf(c.Pipe(tc.PipeName, tc.PipeParams).FetchUntyped(ctx))) + case "sql": + logErr(t, errOf(SQL[map[string]any](ctx, c, tc.SQL))) + case "health": + logErr(t, c.Sys.Health(ctx)) + case "schema_list": + logErr(t, errOf(c.Schema.List(ctx))) + case "schema_refresh": + logErr(t, c.Schema.Refresh(ctx)) + case "policy_get": + logErr(t, errOf(c.Policy.Get(ctx))) + case "policy_set": + logErr(t, c.Policy.Set(ctx, jsonArg[*Policy](t, "policy_body", tc.PolicyBody))) + case "policy_validate": + logErr(t, errOf(c.Policy.Validate(ctx, jsonArg[*Policy](t, "policy_body", tc.PolicyBody)))) + case "dlq_list": + logErr(t, errOf(c.DLQ.List(ctx))) + case "dlq_table": + logErr(t, errOf(c.DLQ.Table(ctx, tc.Table))) + case "pipes_list": + logErr(t, errOf(c.Pipes.List(ctx))) + case "pipes_get": + logErr(t, errOf(c.Pipes.Get(ctx, tc.PipeName))) + case "pipes_set": + logErr(t, c.Pipes.Set(ctx, tc.PipeName, jsonArg[PipeDef](t, "pipe_def", tc.PipeDefBody))) + case "pipes_delete": + logErr(t, c.Pipes.Delete(ctx, tc.PipeName)) + default: + // Hard failure, matching the TS runner: skipped cases break + // cross-SDK parity. + t.Fatalf("unhandled endpoint %q — wire it up in the dispatch switch", tc.Endpoint) + } + + mu.Lock() + defer mu.Unlock() + + wantEq := func(what, want, got string) { + t.Helper() + if want != "" && want != got { + t.Errorf("%s: want %s, got %s", what, want, got) + } + } + wantEq("method", tc.ExpectedMethod, capt.method) + wantEq("content-type", tc.ExpectedContentType, capt.contentType) + // Paths compare by meaning (see normalizePath) but report as written. + if tc.ExpectedPath != "" && normalizePath(tc.ExpectedPath) != normalizePath(capt.path) { + t.Errorf("path: want %s, got %s", tc.ExpectedPath, capt.path) + } + + // Raw body: the NDJSON cases, where the byte layout is the point. + if tc.ExpectedRawBody != nil { + if capt.body != *tc.ExpectedRawBody { + t.Errorf("raw body:\n want: %s\n got: %s", *tc.ExpectedRawBody, capt.body) + } + return + } + if tc.ExpectedBody == nil || string(tc.ExpectedBody) == "null" { + return + } + // Both sides decode into any, so every number is a float64 on both + // sides and map key order is irrelevant to DeepEqual. + var want, got any + if err := json.Unmarshal(tc.ExpectedBody, &want); err != nil { + t.Fatalf("parse expected_body: %v", err) + } + if err := json.Unmarshal([]byte(capt.body), &got); err != nil { + t.Fatalf("parse captured body: %v (body: %s)", err, capt.body) + } + if !reflect.DeepEqual(want, got) { + wantJSON, _ := json.MarshalIndent(want, "", " ") + gotJSON, _ := json.MarshalIndent(got, "", " ") + t.Errorf("body mismatch:\n want: %s\n got: %s", wantJSON, gotJSON) + } + }) + } +} + +// aggOps are the fixture ops that map one-to-one onto a (column, alias) +// aggregation method. Empty args fall through to each method's own defaults. +var aggOps = map[string]func(*QueryBuilder, string, string) *QueryBuilder{ + "count": (*QueryBuilder).Count, + "sum": (*QueryBuilder).Sum, + "avg": (*QueryBuilder).Avg, + "min": (*QueryBuilder).Min, + "max": (*QueryBuilder).Max, + "countDistinct": (*QueryBuilder).CountDistinct, +} + +// applyOps replays the operation chain from the fixture onto a QueryBuilder. +// Fixtures always put select first (mirroring real usage), so rebuilding on +// select is safe and keeps this simple. +func applyOps(t *testing.T, table string, c *Client, ops []wireOp) *QueryBuilder { + t.Helper() + q := c.From(table).Select() + + for _, op := range ops { + if agg, ok := aggOps[op.Method]; ok { + q = agg(q, stringArg(op.Args, 0, ""), stringArg(op.Args, 1, "")) + continue + } + switch op.Method { + case "select": + q = c.From(table).Select(toStringSlice(op.Args)...) + case "selectAll": + q = q.SelectAll() + case "where": + if len(op.Args) != 3 { + t.Fatalf("where needs 3 args, got %d", len(op.Args)) + } + col, colOK := op.Args[0].(string) + rawOp, opOK := op.Args[1].(string) + if !colOK || !opOK { + t.Fatalf("where: want (string, string, any) args, got (%T, %T)", op.Args[0], op.Args[1]) + } + q = q.Where(col, FilterOp(rawOp), op.Args[2]) + case "aggregate": + q = q.Aggregate(stringArg(op.Args, 0, ""), stringArg(op.Args, 1, ""), stringArg(op.Args, 2, "")) + case "groupBy": + q = q.GroupBy(toStringSlice(op.Args)...) + case "orderBy": + q = q.OrderBy(stringArg(op.Args, 0, ""), stringArg(op.Args, 1, "asc")) + case "limit": + q = q.Limit(intArg(op.Args, 0)) + case "timeRange": + q = q.TimeRange(stringArg(op.Args, 0, ""), stringArg(op.Args, 1, ""), stringArg(op.Args, 2, "")) + case "cacheTTL": + q = q.CacheTTL(intArg(op.Args, 0)) + } + } + return q +} + +// insertArg returns the payload for an ingest case. Batch rows arrive from +// JSON as []any; hand Insert the []map[string]any a caller would pass so it +// takes the batch (NDJSON) path a typed slice takes. +func insertArg(t *testing.T, tc wireCase) any { + t.Helper() + if len(tc.Operations) == 0 || tc.Operations[0].Method != "insert" || len(tc.Operations[0].Args) == 0 { + t.Fatalf("ingest case %q needs an insert operation with one arg", tc.Name) + } + arg := tc.Operations[0].Args[0] + batch, ok := arg.([]any) + if !ok { + return arg + } + rows := make([]map[string]any, len(batch)) + for i, r := range batch { + row, ok := r.(map[string]any) + if !ok { + t.Fatalf("fixture row is not an object: %T", r) + } + rows[i] = row + } + return rows +} + +func stringArg(args []any, i int, fallback string) string { + if i >= len(args) { + return fallback + } + s, ok := args[i].(string) + if !ok { + return fallback + } + return s +} + +func intArg(args []any, i int) int { + if i >= len(args) { + return 0 + } + switch v := args[i].(type) { + case float64: + return int(v) + case int: + return v + default: + return 0 + } +} + +func toStringSlice(args []any) []string { + out := make([]string, len(args)) + for i, a := range args { + out[i], _ = a.(string) + } + return out +} + +// normalizePath compares request URIs by meaning: same path, same decoded +// query values regardless of + vs %20 spelling or parameter order. A raw +// string replace would also rewrite literal + characters and stop asserting +// the encoding at all. +func normalizePath(p string) string { + u, err := url.ParseRequestURI(p) + if err != nil { + return p + } + return u.Path + "?" + u.Query().Encode() +} diff --git a/clients/go/dlq.go b/clients/go/dlq.go new file mode 100644 index 00000000..327000ed --- /dev/null +++ b/clients/go/dlq.go @@ -0,0 +1,42 @@ +package wavehouse + +import ( + "context" + "fmt" + "net/url" +) + +// DLQNamespace provides admin-only dead-letter-queue statistics. A deployment +// with the DLQ disabled answers these calls with an [*Error] of Status 404 +// rather than an empty DLQStats, so check Status before reading a zero total. +type DLQNamespace struct { + ctx httpContext + createStream func(table string, opts *StreamOptions) *StreamController +} + +// List returns DLQ statistics (message counts per table). Admin-only. +func (d *DLQNamespace) List(ctx context.Context) (*DLQStats, error) { + return d.stats(ctx, nil) +} + +// Table returns DLQ stats filtered by table name. Admin-only. +func (d *DLQNamespace) Table(ctx context.Context, name string) (*DLQStats, error) { + return d.stats(ctx, url.Values{"table": {name}}) +} + +func (d *DLQNamespace) stats(ctx context.Context, params url.Values) (*DLQStats, error) { + var stats DLQStats + if err := doRequest(ctx, d.ctx, requestOptions{ + method: "GET", + path: "/v1/ops/dlq/stats", + params: params, + }, &stats); err != nil { + return nil, fmt.Errorf("get dlq stats: %w", err) + } + return &stats, nil +} + +// Stream subscribes to live DLQ events. Not yet functional server-side (#197). +func (d *DLQNamespace) Stream(opts *StreamOptions) *StreamController { + return d.createStream("dlq", opts) +} diff --git a/clients/go/e2e_test.go b/clients/go/e2e_test.go new file mode 100644 index 00000000..ef9ae886 --- /dev/null +++ b/clients/go/e2e_test.go @@ -0,0 +1,413 @@ +//go:build e2e + +package wavehouse + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "slices" + "strings" + "testing" + "time" +) + +// e2eClient builds a Client pointing at the live WaveHouse instance. +// It reads WAVEHOUSE_URL (default http://localhost:8080) and the optional +// WAVEHOUSE_AUTH bearer token. The test is skipped when the server is +// unreachable — so `go test -tags e2e` degrades gracefully on a dev +// machine that isn't running the stack. +func e2eClient(t *testing.T) *Client { + t.Helper() + + base := os.Getenv("WAVEHOUSE_URL") + if base == "" { + base = "http://localhost:8080" + } + + cfg := Config{ + BaseURL: base, + Options: &ClientOptions{MaxRetries: Ptr(1)}, + } + if tok := os.Getenv("WAVEHOUSE_AUTH"); tok != "" { + cfg.Auth = StaticToken(tok) + } + + // Probe the server before committing to the test. Bounded so a host that + // accepts the connection but never responds still yields the graceful skip. + probeCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + probe, err := http.NewRequestWithContext(probeCtx, "GET", base+"/v1/health", nil) + if err != nil { + t.Skipf("e2e: bad WAVEHOUSE_URL %q: %v", base, err) + } + resp, err := (&http.Client{Timeout: 3 * time.Second}).Do(probe) + if err != nil { + t.Skipf("e2e: server unreachable at %s: %v", base, err) + } + resp.Body.Close() + + return NewClient(cfg) +} + +// e2eCtx is the context every e2e call uses. The deadlines that matter are +// scoped where they belong: the reachability probe above and waitForRows below. +var e2eCtx = context.Background() + +// marker returns a unique string for the running test, useful for +// inserting distinguishable rows that won't collide across parallel runs. +func marker(t *testing.T) string { + t.Helper() + // Replace slashes in subtest names so it's a clean string value. + safe := strings.ReplaceAll(t.Name(), "/", "_") + return fmt.Sprintf("%s_%d", safe, time.Now().UnixNano()) +} + +// firstTable discovers a usable table from the schema list and returns its +// schema alongside the name. Many E2E tests need a real table to insert/query +// — this avoids hardcoding a name, and returning the schema avoids a second +// Schema.List whose result set might no longer contain the chosen table. +// Sorted so every run picks the same table (map iteration order is random). +func firstTable(t *testing.T, c *Client) (string, TableSchema) { + t.Helper() + schemas, err := c.Schema.List(context.Background()) + if err != nil { + t.Skipf("e2e: cannot list schemas (auth?): %v", err) + } + names := make([]string, 0, len(schemas)) + for name := range schemas { + names = append(names, name) + } + if len(names) == 0 { + t.Skip("e2e: no tables found — server has an empty schema") + } + slices.Sort(names) + return names[0], schemas[names[0]] +} + +// waitForRows polls the marker query until at least want rows are visible or +// the deadline expires. Ingestion is asynchronous — a fixed sleep fails on a +// loaded runner without any real defect. +func waitForRows(t *testing.T, c *Client, table, markerCol, mk string, want int) []map[string]any { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + page, err := c.From(table).Select(markerCol). + Where(markerCol, OpEq, mk). + Limit(max(want, 1)). + FetchUntyped(context.Background()) + if err != nil { + t.Fatalf("query for marker %q: %v", mk, err) + } + if len(page.Data) >= want || time.Now().After(deadline) { + return page.Data + } + time.Sleep(200 * time.Millisecond) + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +func TestE2E_HealthCheck(t *testing.T) { + c := e2eClient(t) + + if err := c.Sys.Health(e2eCtx); err != nil { + t.Fatalf("Health check failed: %v", err) + } +} + +func TestE2E_SchemaList(t *testing.T) { + c := e2eClient(t) + + schemas, err := c.Schema.List(e2eCtx) + if err != nil { + t.Fatalf("Schema.List failed: %v", err) + } + if len(schemas) == 0 { + t.Fatal("Schema.List returned zero tables — expected at least one") + } + // Quick sanity: every table should have columns. + for name, ts := range schemas { + if len(ts.Columns) == 0 { + t.Errorf("table %q has no columns", name) + } + } +} + +// TestE2E_Insert covers both ingest paths against the live server: a bare map +// goes as one JSON body, a slice as an NDJSON batch. +func TestE2E_Insert(t *testing.T) { + tests := []struct { + name string + count int + }{ + {"single row", 1}, + {"batch of three", 3}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := e2eClient(t) + table, ts := firstTable(t, c) + mk := marker(t) + + // Every row carries the same marker, so one query counts them all. + rows := make([]map[string]any, tt.count) + markerCol := "" + for i := range rows { + rows[i], markerCol = buildMarkerRow(t, ts, mk) + } + var payload any = rows + if tt.count == 1 { + payload = rows[0] // a bare map takes the single-insert path + } + + res, err := c.From(table).Insert(e2eCtx, payload) + if err != nil { + t.Fatalf("Insert into %s failed: %v", table, err) + } + if !res.OK { + t.Fatalf("Insert into %s: OK=false", table) + } + + got := waitForRows(t, c, table, markerCol, mk, tt.count) + if len(got) < tt.count { + t.Fatalf("expected >= %d rows for marker %q, got %d", tt.count, mk, len(got)) + } + if v, _ := got[0][markerCol].(string); v != mk { + t.Errorf("marker mismatch: want %q, got %q", mk, v) + } + }) + } +} + +func TestE2E_QueryBuilder(t *testing.T) { + c := e2eClient(t) + table, ts := firstTable(t, c) + + // Pick two columns for a minimal projection. + var cols []string + for _, col := range ts.Columns { + cols = append(cols, col.Name) + if len(cols) >= 2 { + break + } + } + if len(cols) == 0 { + t.Skipf("e2e: table %q has no columns", table) + } + + page, err := c.From(table). + Select(cols...). + OrderBy(cols[0], "asc"). + Limit(5). + FetchUntyped(e2eCtx) + if err != nil { + t.Fatalf("QueryBuilder chain failed: %v", err) + } + // We can't assert exact data, but the chain should execute without error + // and return at most 5 rows. + if len(page.Data) > 5 { + t.Errorf("Limit(5) returned %d rows", len(page.Data)) + } +} + +func TestE2E_TypedFetch(t *testing.T) { + c := e2eClient(t) + table, _ := firstTable(t, c) + + q := c.From(table).SelectAll().Limit(3) + page, err := FetchTyped[map[string]any](e2eCtx, q) + if err != nil { + t.Fatalf("FetchTyped failed: %v", err) + } + // If the table has data we should get rows; if it's empty that's still + // a valid result. The important thing is no error and correct type. + for i, row := range page.Data { + if row == nil { + t.Errorf("row %d is nil", i) + } + } +} + +func TestE2E_SQLQuery(t *testing.T) { + c := e2eClient(t) + + rows, err := SQL[map[string]any](e2eCtx, c, "SELECT 1 AS n") + if err != nil { + skipIfUnauthorized(t, err, "SQL query") + t.Fatalf("SQL query failed: %v", err) + } + if len(rows) != 1 { + t.Fatalf("expected 1 row, got %d", len(rows)) + } + // ClickHouse returns numbers as strings or floats depending on format; + // accept either. + n := rows[0]["n"] + switch v := n.(type) { + case float64: + if v != 1 { + t.Errorf("expected n=1, got %v", v) + } + case string: + if v != "1" { + t.Errorf("expected n=1, got %q", v) + } + default: + t.Errorf("unexpected type for n: %T = %v", n, n) + } +} + +func TestE2E_PolicyGetSet(t *testing.T) { + c := e2eClient(t) + + pol, err := c.Policy.Get(e2eCtx) + if err != nil { + skipIfUnauthorized(t, err, "Policy.Get") + t.Fatalf("Policy.Get failed: %v", err) + } + + // Round-trip: set the same policy back. + if err := c.Policy.Set(e2eCtx, pol); err != nil { + t.Fatalf("Policy.Set (round-trip) failed: %v", err) + } + + // Read again and verify tables still match. + pol2, err := c.Policy.Get(e2eCtx) + if err != nil { + t.Fatalf("Policy.Get (after set) failed: %v", err) + } + if len(pol2.Tables) != len(pol.Tables) { + t.Errorf("policy table count changed: %d -> %d", len(pol.Tables), len(pol2.Tables)) + } +} + +func TestE2E_PipesCRUD(t *testing.T) { + c := e2eClient(t) + + pipeName := fmt.Sprintf("e2e_test_%d", time.Now().UnixNano()) + + // Create + def := PipeDef{ + SQL: "SELECT 1 AS ok", + Description: "E2E test pipe — safe to delete", + } + if err := c.Pipes.Set(e2eCtx, pipeName, def); err != nil { + skipIfUnauthorized(t, err, "Pipes.Set") + t.Fatalf("Pipes.Set (create) failed: %v", err) + } + + // Cleanup: always attempt delete so we don't litter. + t.Cleanup(func() { + _ = c.Pipes.Delete(context.Background(), pipeName) + }) + + // Get + pipe, err := c.Pipes.Get(e2eCtx, pipeName) + if err != nil { + t.Fatalf("Pipes.Get failed: %v", err) + } + if pipe.SQL != def.SQL { + t.Errorf("pipe SQL mismatch: want %q, got %q", def.SQL, pipe.SQL) + } + + // List — verify it appears + pipes, err := c.Pipes.List(e2eCtx) + if err != nil { + t.Fatalf("Pipes.List failed: %v", err) + } + found := false + for _, p := range pipes { + if p.Name == pipeName { + found = true + break + } + } + if !found { + t.Errorf("Pipes.List: created pipe %q not found in list of %d pipes", pipeName, len(pipes)) + } + + // Delete + if err := c.Pipes.Delete(e2eCtx, pipeName); err != nil { + t.Fatalf("Pipes.Delete failed: %v", err) + } + + // Verify gone — Get should fail. + _, err = c.Pipes.Get(e2eCtx, pipeName) + if err == nil { + t.Error("Pipes.Get after delete: expected error, got nil") + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// buildMarkerRow constructs a minimal valid row for the table, injecting the +// marker into the first non-default string column and using sensible values +// for other required columns. It returns the row and the marker column, so +// callers query back the exact column the marker went into. +func buildMarkerRow(t *testing.T, ts TableSchema, mk string) (map[string]any, string) { + t.Helper() + row := make(map[string]any) + markerCol := "" + markerSet := false + for _, col := range ts.Columns { + if col.HasDefault { + continue // let the server fill defaults + } + ct := strings.ToLower(col.Type) + switch { + case strings.Contains(ct, "array(") || strings.Contains(ct, "map(") || + strings.Contains(ct, "tuple(") || strings.Contains(ct, "nested("): + // Container types must be gated BEFORE the substring cases below — + // "array(string)" contains "string" and would otherwise get a + // scalar marker injected into an array column. + t.Skipf("e2e: table %q requires container column %q of type %q", ts.Name, col.Name, col.Type) + case !markerSet && strings.Contains(ct, "string"): + row[col.Name] = mk + markerCol = col.Name + markerSet = true + case strings.Contains(ct, "string"): + row[col.Name] = "e2e" + case strings.Contains(ct, "int"): + row[col.Name] = 0 + case strings.Contains(ct, "float") || strings.Contains(ct, "decimal"): + row[col.Name] = 0.0 + case strings.Contains(ct, "date") || strings.Contains(ct, "datetime"): + row[col.Name] = time.Now().UTC().Format(time.RFC3339) + case strings.Contains(ct, "bool"): + row[col.Name] = false + default: + // No safe synthetic value for this type (UUID, IPv6, ...) — an + // empty string would make the insert fail with a type error that + // looks like an SDK defect. + t.Skipf("e2e: table %q requires column %q of unsupported type %q", ts.Name, col.Name, col.Type) + } + } + if !markerSet { + t.Skipf("e2e: table %q has no non-default string column for marker", ts.Name) + } + return row, markerCol +} + +// skipIfUnauthorized skips the test when err indicates a 401 or 403, +// meaning the operation requires admin auth the current token lacks. +func skipIfUnauthorized(t *testing.T, err error, op string) { + t.Helper() + if isHTTPStatus(err, 401) || isHTTPStatus(err, 403) { + t.Skipf("%s requires admin auth, skipping", op) + } +} + +// isHTTPStatus checks whether err wraps a wavehouse.Error with the given status. +func isHTTPStatus(err error, status int) bool { + var e *Error + if errors.As(err, &e) { + return e.Status == status + } + return false +} diff --git a/clients/go/errors.go b/clients/go/errors.go new file mode 100644 index 00000000..6a321798 --- /dev/null +++ b/clients/go/errors.go @@ -0,0 +1,71 @@ +package wavehouse + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" +) + +// Error is the structured error returned by all SDK operations. Use +// [errors.As] to extract it from wrapped errors. +type Error struct { + Status int `json:"status"` // 0 for network/abort errors + Code string `json:"code"` // e.g. "HTTP_400", "NETWORK_ERROR", "ABORTED", "REDIRECT" + Message string `json:"message"` + Details map[string]any `json:"details,omitempty"` // full parsed error body, when present + Retryable bool `json:"retryable"` +} + +func (e *Error) Error() string { + if e.Status > 0 { + return fmt.Sprintf("wavehouse: %s (%d): %s", e.Code, e.Status, e.Message) + } + return fmt.Sprintf("wavehouse: %s: %s", e.Code, e.Message) +} + +// IsRetryable reports whether err wraps a retryable [*Error]. +func IsRetryable(err error) bool { + var e *Error + return errors.As(err, &e) && e.Retryable +} + +// parseErrorResponse creates an Error from an HTTP response. +func parseErrorResponse(res *http.Response) *Error { + var body map[string]any + if res.Body != nil { + raw, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20)) // cap at 1 MiB + _ = json.Unmarshal(raw, &body) + } + + msg := http.StatusText(res.StatusCode) + if s, ok := body["error"].(string); ok { + msg = s + } else if s, ok := body["message"].(string); ok { + msg = s + } + + retryable := res.StatusCode >= 500 || res.StatusCode == http.StatusTooManyRequests + return &Error{ + Status: res.StatusCode, + Code: fmt.Sprintf("HTTP_%d", res.StatusCode), + Message: msg, + Details: body, + Retryable: retryable, + } +} + +// networkError creates an Error from a transport-level failure. +func networkError(cause error) *Error { + msg := "unknown network error" + if cause != nil { + msg = cause.Error() + } + return &Error{ + Status: 0, + Code: "NETWORK_ERROR", + Message: msg, + Retryable: true, + } +} diff --git a/clients/go/errors_test.go b/clients/go/errors_test.go new file mode 100644 index 00000000..459cb610 --- /dev/null +++ b/clients/go/errors_test.go @@ -0,0 +1,129 @@ +package wavehouse + +import ( + "errors" + "io" + "net/http" + "strings" + "testing" +) + +func TestParseErrorResponse(t *testing.T) { + tests := []struct { + name string + status int + body string + wantMsg string + wantCode string + wantRetry bool + nilDetails bool + }{ + { + name: "JSONError", + status: 404, + body: `{"error":"unknown table: foo"}`, + wantMsg: "unknown table: foo", + wantCode: "HTTP_404", + }, + { + name: "MessageField", + status: 400, + body: `{"message":"bad request"}`, + wantMsg: "bad request", + }, + { + name: "FallsBackToStatusText", + status: 500, + body: `{"code":123}`, + wantMsg: "Internal Server Error", + wantRetry: true, + }, + { + name: "NonJSONBody", + status: 502, + body: "plain text", + wantMsg: "Bad Gateway", + wantRetry: true, + nilDetails: true, + }, + // Retryability is decided by status class alone — 429 and 5xx only. + {name: "ForbiddenNotRetryable", status: 403, body: `{"error":"nope"}`, wantMsg: "nope"}, + {name: "TooManyRequestsRetryable", status: 429, body: `{"error":"slow down"}`, wantMsg: "slow down", wantRetry: true}, + {name: "ServiceUnavailableRetryable", status: 503, body: `{"error":"down"}`, wantMsg: "down", wantRetry: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + res := &http.Response{ + StatusCode: tt.status, + Body: io.NopCloser(strings.NewReader(tt.body)), + Header: http.Header{}, + } + e := parseErrorResponse(res) + if e.Status != tt.status { + t.Fatalf("want status %d, got %d", tt.status, e.Status) + } + if tt.wantCode != "" && e.Code != tt.wantCode { + t.Fatalf("want code %s, got %s", tt.wantCode, e.Code) + } + if e.Message != tt.wantMsg { + t.Fatalf("want message %q, got %q", tt.wantMsg, e.Message) + } + if e.Retryable != tt.wantRetry { + t.Fatalf("want retryable=%v, got %v", tt.wantRetry, e.Retryable) + } + if tt.nilDetails && e.Details != nil { + t.Fatal("details should be nil") + } + }) + } +} + +func TestNetworkError(t *testing.T) { + e := networkError(errors.New("connection refused")) + if e.Code != "NETWORK_ERROR" { + t.Fatalf("want NETWORK_ERROR, got %s", e.Code) + } + if e.Message != "connection refused" { + t.Fatalf("want 'connection refused', got %s", e.Message) + } + if !e.Retryable { + t.Fatal("network errors should be retryable") + } + if e.Status != 0 { + t.Fatalf("want status 0, got %d", e.Status) + } +} + +func TestError_ErrorMethod(t *testing.T) { + for _, e := range []*Error{ + {Status: 404, Code: "HTTP_404", Message: "not found"}, + {Status: 0, Code: "NETWORK_ERROR", Message: "timeout"}, + } { + if got := e.Error(); !strings.Contains(got, e.Code) || !strings.Contains(got, e.Message) { + t.Errorf("Error() = %q, want it to name both %q and %q", got, e.Code, e.Message) + } + } +} + +func TestIsRetryable(t *testing.T) { + if !IsRetryable(&Error{Retryable: true}) { + t.Fatal("want true for retryable error") + } + if IsRetryable(&Error{Retryable: false}) { + t.Fatal("want false for non-retryable error") + } + if IsRetryable(errors.New("plain error")) { + t.Fatal("want false for non-wavehouse error") + } +} + +func TestErrorsAs(t *testing.T) { + err := error(&Error{Status: 403, Code: "HTTP_403", Message: "forbidden"}) + var e *Error + if !errors.As(err, &e) { + t.Fatal("errors.As should find *Error") + } + if e.Status != 403 { + t.Fatalf("want 403, got %d", e.Status) + } +} diff --git a/clients/go/example_test.go b/clients/go/example_test.go new file mode 100644 index 00000000..b8e566c7 --- /dev/null +++ b/clients/go/example_test.go @@ -0,0 +1,75 @@ +package wavehouse_test + +import ( + "context" + "fmt" + "log" + + wavehouse "github.com/Wave-RF/WaveHouse/clients/go" +) + +// ExampleNewClient demonstrates creating an unauthenticated client and +// performing a health check. The Output assertion is omitted because the +// example needs a running server. +func ExampleNewClient() { + client := wavehouse.NewClient(wavehouse.Config{ + BaseURL: "http://localhost:8080", + }) + + // Health check — returns nil when the server is reachable. + if err := client.Sys.Health(context.Background()); err != nil { + log.Fatal(err) + } +} + +func ExampleNewClient_withAuth() { + _ = wavehouse.NewClient(wavehouse.Config{ + BaseURL: "http://localhost:8080", + Auth: wavehouse.StaticToken("my-jwt-token"), + }) +} + +func ExampleClient_From() { + client := wavehouse.NewClient(wavehouse.Config{ + BaseURL: "http://localhost:8080", + }) + + // Insert a row. + _, _ = client.From("clicks").Insert(context.Background(), map[string]any{ + "page": "/home", + "button": "cta", + }) + + // Query with the builder. + page, err := client.From("clicks"). + Select("page", "button"). + Where("page", wavehouse.OpEq, "/home"). + OrderBy("page", "asc"). + Limit(10). + FetchUntyped(context.Background()) + if err != nil { + log.Fatal(err) + } + + for _, row := range page.Data { + fmt.Println(row["page"]) + } +} + +func ExampleSQL() { + client := wavehouse.NewClient(wavehouse.Config{ + BaseURL: "http://localhost:8080", + Auth: wavehouse.StaticToken("admin-token"), + }) + + rows, err := wavehouse.SQL[map[string]any]( + context.Background(), client, + "SELECT page, count() as views FROM clicks GROUP BY page LIMIT 5", + ) + if err != nil { + log.Fatal(err) + } + for _, row := range rows { + fmt.Println(row["page"], row["views"]) + } +} diff --git a/clients/go/go.mod b/clients/go/go.mod new file mode 100644 index 00000000..b78eaa50 --- /dev/null +++ b/clients/go/go.mod @@ -0,0 +1,6 @@ +module github.com/Wave-RF/WaveHouse/clients/go + +// Library floor, deliberately lower than the server's pinned toolchain: +// the newest things this module uses are range-over-int and math/rand/v2 +// (Go 1.22). Keep it a supported-releases floor, not a patch pin. +go 1.24 diff --git a/clients/go/http.go b/clients/go/http.go new file mode 100644 index 00000000..b8657acc --- /dev/null +++ b/clients/go/http.go @@ -0,0 +1,248 @@ +package wavehouse + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "math" + "math/rand/v2" + "net/http" + "net/url" + "strconv" + "time" +) + +var errAborted = &Error{Status: 0, Code: "ABORTED", Message: "Request aborted", Retryable: false} + +// maxRetryAfter caps server-supplied Retry-After delays so a hostile or +// misconfigured server can't park the calling goroutine for hours. +const maxRetryAfter = 30 * time.Second + +// httpContext carries per-client state needed by every request. +type httpContext struct { + baseURL string + auth func(ctx context.Context) (string, error) + maxRetries int + httpClient *http.Client + headers map[string]string +} + +// applyConfiguredHeaders writes the client's configured headers onto a request. +// Call it before the SDK sets its own headers: Set replaces, so the SDK's +// later writes win any collision. +func applyConfiguredHeaders(h http.Header, configured map[string]string) { + for k, v := range configured { + h.Set(k, v) + } +} + +// requestOptions describes a single HTTP request. +type requestOptions struct { + method string + path string + body any // JSON-serialized if non-nil + rawBody string // sent verbatim if non-empty (takes precedence over body) + contentType string // overrides Content-Type (default "application/json") + params url.Values +} + +// doRequest issues a request with auth, retry and backoff, decoding the +// response body into dst unless dst is nil. +func doRequest(ctx context.Context, hctx httpContext, opts requestOptions, dst any) error { + reqURL := buildURL(hctx.baseURL, opts.path, opts.params) + ct := opts.contentType + if ct == "" { + ct = "application/json" + } + + // Serialize body once so every retry sends identical bytes. + var bodyBytes []byte + if opts.rawBody != "" { + bodyBytes = []byte(opts.rawBody) + } else if opts.body != nil { + var err error + bodyBytes, err = json.Marshal(opts.body) + if err != nil { + return fmt.Errorf("wavehouse: marshal request body: %w", err) + } + } + + // Resolve auth once per request (not per attempt). + var authHeader string + if hctx.auth != nil { + token, err := hctx.auth(ctx) + if err != nil { + return fmt.Errorf("wavehouse: auth provider: %w", err) + } + if token != "" { + authHeader = "Bearer " + token + } + } + + // Never follow a redirect while carrying a credential: net/http drops + // Authorization across hosts but forwards custom headers verbatim, so a 302 + // would hand a configured X-Operator-Key to whatever Location names. The + // SSE path takes the same stance (SSE_REDIRECT in stream.go). Copied so a + // caller-supplied client keeps its own CheckRedirect. + client := hctx.httpClient + credentialed := authHeader != "" || len(hctx.headers) > 0 + if credentialed { + c := *client + c.CheckRedirect = func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + } + client = &c + } + + var lastErr error + maxAttempts := hctx.maxRetries + 1 + + // Retries every method including POST: /v1/ingest documents at-least-once + // delivery, with server-side dedup as the safety net. + for attempt := range maxAttempts { + var bodyReader io.Reader + if bodyBytes != nil { + bodyReader = bytes.NewReader(bodyBytes) + } + + req, err := http.NewRequestWithContext(ctx, opts.method, reqURL, bodyReader) + if err != nil { + return fmt.Errorf("wavehouse: build request: %w", err) + } + applyConfiguredHeaders(req.Header, hctx.headers) + req.Header.Set("Content-Type", ct) + req.Header.Set("Accept", "application/json") + if authHeader != "" { + req.Header.Set("Authorization", authHeader) + } + + res, err := client.Do(req) + if err != nil { + if ctx.Err() != nil { + return errAborted + } + lastErr = networkError(err) + if attempt < maxAttempts-1 { + if sleepErr := sleepWithContext(ctx, backoff(attempt)); sleepErr != nil { + return errAborted + } + } + continue + } + + if res.StatusCode >= 200 && res.StatusCode < 300 { + defer func() { _ = res.Body.Close() }() + if dst == nil { + _, _ = io.Copy(io.Discard, res.Body) + return nil + } + raw, readErr := io.ReadAll(res.Body) + if readErr != nil { + return networkError(readErr) + } + if len(raw) == 0 { + return nil + } + if err := json.Unmarshal(raw, dst); err != nil { + return &Error{ + Status: 0, + Code: "NETWORK_ERROR", + Message: fmt.Errorf("decode response: %w", err).Error(), + Retryable: false, + } + } + return nil + } + + if credentialed && res.StatusCode >= 300 && res.StatusCode < 400 { + loc := res.Header.Get("Location") + _ = res.Body.Close() + return &Error{ + Status: res.StatusCode, + Code: "REDIRECT", + Message: fmt.Sprintf( + "request redirected to %q and the SDK did not follow it; redirects are refused while the request carries a credential", + loc), + Retryable: false, + } + } + + apiErr := parseErrorResponse(res) + _ = res.Body.Close() + + // 503/429 with Retry-After: wait the header's duration, capped. + if res.StatusCode == http.StatusServiceUnavailable || res.StatusCode == http.StatusTooManyRequests { + if ra := res.Header.Get("Retry-After"); ra != "" && attempt < maxAttempts-1 { + if sleepErr := sleepWithContext(ctx, retryAfterDelay(ra, attempt)); sleepErr != nil { + return errAborted + } + lastErr = apiErr + continue + } + } + + if apiErr.Retryable && attempt < maxAttempts-1 { + if sleepErr := sleepWithContext(ctx, backoff(attempt)); sleepErr != nil { + return errAborted + } + lastErr = apiErr + continue + } + + return apiErr + } + + return lastErr +} + +func buildURL(base, path string, params url.Values) string { + u := base + path + if len(params) > 0 { + u += "?" + params.Encode() + } + return u +} + +// retryAfterDelay resolves a Retry-After header (delta-seconds or HTTP-date) +// into a wait, clamped to maxRetryAfter. An unparseable header falls back to +// the ordinary backoff for this attempt, not the maximum. +func retryAfterDelay(ra string, attempt int) time.Duration { + delay := backoff(attempt) + if secs, err := strconv.Atoi(ra); err == nil && secs > 0 { + // Compare before converting: secs*time.Second wraps negative past + // ~9.2e9 seconds and min() below would then fire the timer instantly. + if secs > int(maxRetryAfter/time.Second) { + return maxRetryAfter + } + delay = time.Duration(secs) * time.Second + } else if parsed, err := http.ParseTime(ra); err == nil { + if d := time.Until(parsed); d > 0 { + delay = d + } + } + return min(delay, maxRetryAfter) +} + +func backoff(attempt int) time.Duration { + ms := 1000 * math.Pow(2, float64(attempt)) + // ±20% jitter so simultaneous failures don't retry in lockstep; capped + // after jitter so the documented 30s maximum holds. + ms *= 0.8 + 0.4*rand.Float64() //nolint:gosec // retry jitter, not cryptographic + return time.Duration(min(ms, 30000)) * time.Millisecond +} + +func sleepWithContext(ctx context.Context, d time.Duration) error { + if ctx.Err() != nil { + return ctx.Err() + } + t := time.NewTimer(d) + defer t.Stop() + select { + case <-t.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/clients/go/http_test.go b/clients/go/http_test.go new file mode 100644 index 00000000..ca5423a9 --- /dev/null +++ b/clients/go/http_test.go @@ -0,0 +1,528 @@ +package wavehouse + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "net/url" + "sync/atomic" + "testing" + "time" +) + +// errIs checks if err wraps a *Error with the given code. +func errIs(err error, code string) bool { + var e *Error + if errors.As(err, &e) { + return e.Code == code + } + return false +} + +// testCtx and queryTestCtx are the two entry points every test in this package +// uses to reach a throwaway server: the bare transport context, and a full +// Client wired to the same server. +func testCtx(t *testing.T, handler http.Handler) httpContext { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + return httpContext{ + baseURL: srv.URL, + maxRetries: 0, + httpClient: srv.Client(), + } +} + +func queryTestCtx(t *testing.T, handler http.Handler) *Client { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + return NewClient(Config{ + BaseURL: srv.URL, + HTTPClient: srv.Client(), + Options: &ClientOptions{MaxRetries: Ptr(0)}, + }) +} + +// recordedRequest is one request as the test server saw it — everything a test +// might assert on, copied on the server goroutine. +type recordedRequest struct { + method string + path string + query url.Values + header http.Header + body []byte +} + +// jsonBody decodes the recorded body as a JSON object. UseNumber keeps int64 +// cursor values exact on the assertion side too. +func (r recordedRequest) jsonBody(t *testing.T) map[string]any { + t.Helper() + dec := json.NewDecoder(bytes.NewReader(r.body)) + dec.UseNumber() + var m map[string]any + if err := dec.Decode(&m); err != nil { + t.Fatalf("decode request body %q: %v", r.body, err) + } + return m +} + +// recordRequests wraps handler so every request it serves lands on the +// returned buffered channel — a channel, not a shared variable, so -race sees +// the edge between the server goroutine and the test. handler still reads an +// intact Body. +func recordRequests(t *testing.T, handler http.Handler) (http.Handler, <-chan recordedRequest) { + t.Helper() + seen := make(chan recordedRequest, 16) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("read request body: %v", err) + } + seen <- recordedRequest{ + method: r.Method, + path: r.URL.Path, + query: r.URL.Query(), + header: r.Header.Clone(), + body: raw, + } + r.Body = io.NopCloser(bytes.NewReader(raw)) + handler.ServeHTTP(w, r) + }), seen +} + +func recordingCtx(t *testing.T, handler http.Handler) (httpContext, <-chan recordedRequest) { + t.Helper() + h, seen := recordRequests(t, handler) + return testCtx(t, h), seen +} + +func recordingClient(t *testing.T, handler http.Handler) (*Client, <-chan recordedRequest) { + t.Helper() + h, seen := recordRequests(t, handler) + return queryTestCtx(t, h), seen +} + +// ok200 answers with a bare 200; jsonArray with an empty JSON array — the +// minimal valid reply for a list endpoint. +var ( + ok200 = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + jsonArray = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `[]`) + }) +) + +// jsonResponse answers any request with body encoded as JSON. +func jsonResponse(body any) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(body) + } +} + +func TestDoRequest_SuccessfulGET(t *testing.T) { + hctx := testCtx(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) + })) + + var result map[string]string + err := doRequest(context.Background(), hctx, requestOptions{ + method: "GET", + path: "/health", + }, &result) + if err != nil { + t.Fatal(err) + } + if result["status"] != "ok" { + t.Fatalf("want ok, got %v", result) + } +} + +// TestDoRequest_RequestShape: what the transport puts on the wire for each +// body/auth combination. +func TestDoRequest_RequestShape(t *testing.T) { + tests := []struct { + name string + auth func(context.Context) (string, error) + opts requestOptions + wantCT string + wantBody string + wantAuth string + }{ + { + name: "a struct body is marshaled as JSON", + opts: requestOptions{method: "POST", path: "/v1/ingest", body: map[string]string{"page": "/home"}}, + wantCT: "application/json", + wantBody: `{"page":"/home"}`, + }, + { + name: "a raw body keeps the caller's content type", + opts: requestOptions{method: "POST", path: "/v1/ingest", rawBody: `{"page":"/a"}`, contentType: "application/x-ndjson"}, + wantCT: "application/x-ndjson", + wantBody: `{"page":"/a"}`, + }, + { + name: "the auth provider's token becomes a Bearer header", + auth: StaticToken("my-token"), + opts: requestOptions{method: "GET", path: "/v1/ops/schema"}, + wantCT: "application/json", + wantAuth: "Bearer my-token", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hctx, reqs := recordingCtx(t, ok200) + hctx.auth = tt.auth + + if err := doRequest(context.Background(), hctx, tt.opts, nil); err != nil { + t.Fatal(err) + } + got := <-reqs + if got.method != tt.opts.method || got.path != tt.opts.path { + t.Fatalf("want %s %s, got %s %s", tt.opts.method, tt.opts.path, got.method, got.path) + } + if ct := got.header.Get("Content-Type"); ct != tt.wantCT { + t.Fatalf("want Content-Type %q, got %q", tt.wantCT, ct) + } + if string(got.body) != tt.wantBody { + t.Fatalf("want body %q, got %q", tt.wantBody, got.body) + } + if auth := got.header.Get("Authorization"); auth != tt.wantAuth { + t.Fatalf("want Authorization %q, got %q", tt.wantAuth, auth) + } + }) + } +} + +// TestDoRequest_RetryPolicy: which statuses are retried, how many attempts +// they take, and whether Retry-After is honored. +func TestDoRequest_RetryPolicy(t *testing.T) { + tests := []struct { + name string + status int + retryAfter string + maxRetries int + wantAttempts int32 + wantErrCode string + wantMinElapsed time.Duration + }{ + {name: "4xx is not retried", status: 404, maxRetries: 2, wantAttempts: 1, wantErrCode: "HTTP_404"}, + {name: "5xx retries until it succeeds", status: 500, maxRetries: 2, wantAttempts: 3}, + { + name: "429 waits out Retry-After", status: 429, retryAfter: "1", + maxRetries: 1, wantAttempts: 2, wantMinElapsed: 900 * time.Millisecond, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var calls atomic.Int32 + hctx := testCtx(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) >= tt.wantAttempts && tt.wantErrCode == "" { + _ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) + return + } + w.Header().Set("Content-Type", "application/json") + if tt.retryAfter != "" { + w.Header().Set("Retry-After", tt.retryAfter) + } + w.WriteHeader(tt.status) + _ = json.NewEncoder(w).Encode(map[string]string{"error": "boom"}) + })) + hctx.maxRetries = tt.maxRetries + + start := time.Now() + var result map[string]string + err := doRequest(context.Background(), hctx, requestOptions{method: "GET", path: "/health"}, &result) + if tt.wantErrCode == "" && err != nil { + t.Fatalf("want success after retries, got %v", err) + } + if tt.wantErrCode != "" && !errIs(err, tt.wantErrCode) { + t.Fatalf("want %s error, got %v", tt.wantErrCode, err) + } + if got := calls.Load(); got != tt.wantAttempts { + t.Fatalf("want %d attempts, got %d", tt.wantAttempts, got) + } + if elapsed := time.Since(start); elapsed < tt.wantMinElapsed { + t.Fatalf("Retry-After not honored — retried after only %v", elapsed) + } + }) + } +} + +func TestDoRequest_AbortedOnCancel(t *testing.T) { + hctx := testCtx(t, http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + time.Sleep(5 * time.Second) + })) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel immediately + + err := doRequest(ctx, hctx, requestOptions{method: "GET", path: "/health"}, nil) + if !errIs(err, "ABORTED") { + t.Fatalf("want ABORTED, got %v", err) + } +} + +func TestDoRequest_EmptyResponse(t *testing.T) { + var result map[string]string + err := doRequest(context.Background(), testCtx(t, ok200), requestOptions{ + method: "POST", + path: "/v1/ops/schema/refresh", + }, &result) + if err != nil { + t.Fatal(err) + } + // Empty body = no decode, result stays zero value. + if result != nil { + t.Fatalf("want nil, got %v", result) + } +} + +func TestBackoff(t *testing.T) { + tests := []struct { + name string + attempt int + base time.Duration + }{ + {"Attempt0", 0, 1 * time.Second}, + {"Attempt1", 1, 2 * time.Second}, + {"Attempt2", 2, 4 * time.Second}, + {"Attempt3", 3, 8 * time.Second}, + {"CappedAt30s", 10, 30 * time.Second}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := backoff(tt.attempt) + if tt.name == "CappedAt30s" { + // The cap is applied *after* jitter, so this is exact — a ±20% + // window here would also accept capping before the jitter, + // which lets the documented 30s max drift to 36s. + if got != 30*time.Second { + t.Errorf("backoff(%d) = %v, want exactly 30s", tt.attempt, got) + } + return + } + // backoff applies ±20% jitter around the exponential base. + lo := time.Duration(float64(tt.base) * 0.8) + hi := time.Duration(float64(tt.base) * 1.2) + if got < lo || got > hi { + t.Errorf("backoff(%d) = %v, want within [%v, %v]", tt.attempt, got, lo, hi) + } + }) + } +} + +func TestRetryAfterDelay(t *testing.T) { + tests := []struct { + name string + ra string + want time.Duration + }{ + {"DeltaSeconds", "5", 5 * time.Second}, + {"ClampedToMax", "3600", maxRetryAfter}, + // time.Duration(secs) * time.Second wraps negative past ~9.2e9s; an + // unguarded min() then picks the negative and retries instantly. + {"OverflowClamped", "10000000000", maxRetryAfter}, + {"MaxIntClamped", "9223372036854775807", maxRetryAfter}, + {"HTTPDateFuture", time.Now().Add(10 * time.Second).UTC().Format(http.TimeFormat), 0}, // range-checked below + {"Garbage", "not-a-delay", 0}, // range-checked below + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := retryAfterDelay(tt.ra, 0) + switch tt.name { + case "HTTPDateFuture": + // Lower bound well clear of the backoff(0) fallback (~1s), so a + // broken HTTP-date branch can't pass by falling through to it. + if got < 5*time.Second || got > 10*time.Second { + t.Fatalf("want ~10s, got %v", got) + } + case "Garbage": + // Falls back to backoff(0): 1s ±20% jitter. + if got < 800*time.Millisecond || got > 1200*time.Millisecond { + t.Fatalf("want backoff(0) fallback, got %v", got) + } + default: + if got != tt.want { + t.Fatalf("want %v, got %v", tt.want, got) + } + } + }) + } +} + +// A BaseURL carrying a path prefix must survive on both transports — the bug +// #428 fixed in the TS client, which Go avoids by concatenating rather than +// resolving. Guards against a future switch to url.JoinPath/ResolveReference. +func TestBaseURLPathPrefixIsPreserved(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/warehouse/v1/query", func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) + }) + srv := httptest.NewServer(mux) // anything off-prefix 404s + t.Cleanup(srv.Close) + + for _, base := range []string{srv.URL + "/api/warehouse", srv.URL + "/api/warehouse/"} { + client := NewClient(Config{BaseURL: base, Options: &ClientOptions{}, HTTPClient: srv.Client()}) + + var result map[string]string + if err := doRequest(context.Background(), client.ctx, requestOptions{ + method: "POST", + path: "/v1/query", + }, &result); err != nil { + t.Fatalf("base %q: %v", base, err) + } + if result["status"] != "ok" { + t.Fatalf("base %q: want ok, got %v", base, result) + } + } +} + +// headerCaptureClient is a Client with configured headers and auth pointed at +// a server that records what it received. +func headerCaptureClient(t *testing.T, opts *ClientOptions, auth func(context.Context) (string, error)) (*Client, <-chan recordedRequest) { + t.Helper() + h, reqs := recordRequests(t, jsonArray) + srv := httptest.NewServer(h) + t.Cleanup(srv.Close) + return NewClient(Config{BaseURL: srv.URL, Auth: auth, HTTPClient: srv.Client(), Options: opts}), reqs +} + +// TestConfiguredHeadersOnRESTRequests: ClientOptions.Headers apply to every +// REST call, are matched case-insensitively, and always lose to the SDK's own +// headers rather than appending alongside them. +func TestConfiguredHeadersOnRESTRequests(t *testing.T) { + tests := []struct { + name string + configured map[string]string + auth func(context.Context) (string, error) + header string + want string + }{ + { + name: "custom header is forwarded", + configured: map[string]string{"X-Operator-Key": "op-secret"}, + header: "X-Operator-Key", + want: "op-secret", + }, + { + name: "name matching is case-insensitive", + configured: map[string]string{"x-tenant-id": "acme"}, + header: "X-Tenant-Id", + want: "acme", + }, + { + name: "SDK Accept outranks a configured one", + configured: map[string]string{"Accept": "text/plain"}, + header: "Accept", + want: "application/json", + }, + { + name: "SDK Authorization outranks a configured one", + configured: map[string]string{"Authorization": "Bearer configured"}, + auth: StaticToken("real-token"), + header: "Authorization", + want: "Bearer real-token", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + client, reqs := headerCaptureClient(t, &ClientOptions{Headers: tc.configured}, tc.auth) + if _, err := client.Schema.List(context.Background()); err != nil { + t.Fatalf("schema list: %v", err) + } + got := (<-reqs).header + if v := got.Values(tc.header); len(v) != 1 { + t.Fatalf("want exactly one %s header, got %v", tc.header, v) + } + if v := got.Get(tc.header); v != tc.want { + t.Fatalf("want %s: %q, got %q", tc.header, tc.want, v) + } + }) + } +} + +// TestConfiguredHeadersAreCopied: mutating the caller's map after NewClient +// must not change what later requests send. +func TestConfiguredHeadersAreCopied(t *testing.T) { + headers := map[string]string{"X-Tenant-Id": "acme"} + client, reqs := headerCaptureClient(t, &ClientOptions{Headers: headers}, nil) + headers["X-Tenant-Id"] = "attacker" + delete(headers, "X-Tenant-Id") + + if _, err := client.Schema.List(context.Background()); err != nil { + t.Fatalf("schema list: %v", err) + } + if v := (<-reqs).header.Get("X-Tenant-Id"); v != "acme" { + t.Fatalf("want the value captured at construction, got %q", v) + } +} + +// TestRESTRedirectsWithCredentials pins the REST half of the invariant that +// TestStream_TerminalConnectFailures pins for SSE: net/http strips only its own +// four sensitive headers across hosts and forwards configured ones verbatim, so +// following a 3xx while credentialed would hand X-Operator-Key (or a bearer +// token, on a same-host hop) to whatever Location names. +func TestRESTRedirectsWithCredentials(t *testing.T) { + // The redirect target records anything that reaches it. Nothing should. + var leaked http.Header + dst := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + leaked = r.Header.Clone() + _, _ = io.WriteString(w, `[]`) + })) + defer dst.Close() + + redirector := func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, dst.URL+r.URL.Path, http.StatusFound) + } + + tests := []struct { + name string + opts *ClientOptions + auth func(context.Context) (string, error) + wantErr bool + }{ + {name: "configured header", opts: &ClientOptions{Headers: map[string]string{"X-Operator-Key": "secret"}}, wantErr: true}, + {name: "bearer token", auth: StaticToken("secret"), wantErr: true}, + // Nothing to protect, so the ordinary redirect-following stands. + {name: "no credential follows the redirect", wantErr: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + leaked = nil + src := httptest.NewServer(http.HandlerFunc(redirector)) + defer src.Close() + + client := NewClient(Config{BaseURL: src.URL, Auth: tt.auth, Options: tt.opts, HTTPClient: src.Client()}) + _, err := client.Schema.List(context.Background()) + + if !tt.wantErr { + if err != nil { + t.Fatalf("uncredentialed request should follow the redirect, got %v", err) + } + return + } + var apiErr *Error + if !errors.As(err, &apiErr) { + t.Fatalf("want a *wavehouse.Error, got %v", err) + } + if apiErr.Code != "REDIRECT" { + t.Errorf("want code REDIRECT, got %q (%v)", apiErr.Code, apiErr) + } + if apiErr.Retryable { + t.Error("a refused redirect is terminal, not retryable") + } + if leaked != nil { + t.Errorf("credentialed request reached the redirect target: %v", leaked) + } + }) + } +} diff --git a/clients/go/live_query.go b/clients/go/live_query.go new file mode 100644 index 00000000..715ca63d --- /dev/null +++ b/clients/go/live_query.go @@ -0,0 +1,154 @@ +package wavehouse + +import ( + "context" + "sync" + "time" +) + +// LiveQueryHandle controls a live query that combines historical backfill +// with a real-time stream. +type LiveQueryHandle struct { + stream *StreamController + cancel context.CancelFunc + unsub func() + closeOnce sync.Once + + mu sync.Mutex + buffer []StreamEvent + buffering bool + closed bool +} + +// newLiveQuery buffers stream events while the historical fetch runs, then +// replays the buffer minus anything the backfill already delivered. +func newLiveQuery( + stream *StreamController, + fetchFn func(ctx context.Context) ([]map[string]any, error), + sub *StreamSubscriber, +) *LiveQueryHandle { + ctx, cancel := context.WithCancel(context.Background()) //nolint:gosec // cancel is called in Close() + lq := &LiveQueryHandle{ + stream: stream, + cancel: cancel, + buffering: true, + } + + // User callbacks are invoked outside lq.mu so a subscriber may call Close + // without deadlocking. + lq.unsub = stream.Subscribe(&StreamSubscriber{ + Next: func(event StreamEvent) { + lq.mu.Lock() + if lq.closed { + lq.mu.Unlock() + return + } + if lq.buffering { + lq.buffer = append(lq.buffer, event) + lq.mu.Unlock() + return + } + lq.mu.Unlock() + if sub.Next != nil { + sub.Next(event) + } + }, + Status: func(s StreamStatus) { + if !lq.isClosed() && sub.Status != nil { + sub.Status(s) + } + }, + Error: func(err error) { + if !lq.isClosed() && sub.Error != nil { + sub.Error(err) + } + }, + }) + + go func() { + rows, err := fetchFn(ctx) + if ctx.Err() != nil || lq.isClosed() { + return + } + + if sub.Initial != nil { + sub.Initial(rows, err) + } + + if err != nil { + lq.mu.Lock() + lq.buffering = false + lq.buffer = nil + lq.mu.Unlock() + return + } + + // Dedup bound: the maximum backfilled timestamp as a parsed time. The + // last row can be the oldest, and RFC3339 strings with varying + // fractional digits do not sort lexically. + var lastTS time.Time + for _, row := range rows { + if s, ok := row["received_timestamp"].(string); ok { + if ts, perr := time.Parse(time.RFC3339Nano, s); perr == nil && ts.After(lastTS) { + lastTS = ts + } + } + } + + // buffering stays true until the buffer is provably empty under the + // lock, which keeps delivery ordered and single-threaded. + for { + lq.mu.Lock() + if lq.closed { + lq.mu.Unlock() + return + } + pending := lq.buffer + lq.buffer = nil + if len(pending) == 0 { + lq.buffering = false + lq.mu.Unlock() + break + } + lq.mu.Unlock() + + for _, event := range pending { + if lq.isClosed() { + return + } + // Skip events already delivered in the backfill. + if !lastTS.IsZero() { + if ts, perr := time.Parse(time.RFC3339Nano, event.Timestamp); perr == nil && !ts.After(lastTS) { + continue + } + } + if sub.Next != nil { + sub.Next(event) + } + } + } + }() + + return lq +} + +func (lq *LiveQueryHandle) isClosed() bool { + lq.mu.Lock() + defer lq.mu.Unlock() + return lq.closed +} + +// Close shuts down the live query and the underlying stream. The close state +// is applied synchronously: no new subscriber callbacks start after Close +// returns (a callback already in flight may still complete). +func (lq *LiveQueryHandle) Close() { + lq.closeOnce.Do(func() { + lq.mu.Lock() + lq.closed = true + lq.buffer = nil + lq.mu.Unlock() + lq.unsub() + lq.cancel() + lq.stream.Close() + }) +} diff --git a/clients/go/live_query_test.go b/clients/go/live_query_test.go new file mode 100644 index 00000000..c171b198 --- /dev/null +++ b/clients/go/live_query_test.go @@ -0,0 +1,158 @@ +package wavehouse + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" +) + +// bareStream builds a StreamController that never dials anything — events are +// injected with emitEvent, exactly how the run loop feeds real ones. +func bareStream() *StreamController { + return &StreamController{ + status: StatusLive, + eventCh: make(chan StreamEvent, 16), + done: make(chan struct{}), + cancel: func() {}, + } +} + +func liveEvent(ts string) StreamEvent { + return StreamEvent{Table: "clicks", Timestamp: ts, Data: map[string]any{"page": "/home"}} +} + +func awaitInitial(t *testing.T, ch <-chan []map[string]any) []map[string]any { + t.Helper() + select { + case rows := <-ch: + return rows + case <-time.After(5 * time.Second): + t.Fatal("Initial never fired") + return nil + } +} + +// TestLiveQuery_BackfillThenLive: events arriving while the backfill is in +// flight are buffered, flushed once Initial has fired, and deduplicated +// against the *maximum* backfilled timestamp — not the last row, which a +// descending sort makes the oldest. +func TestLiveQuery_BackfillThenLive(t *testing.T) { + tests := []struct { + name string + backfill []string // received_timestamp per backfilled row, in server order + emit []string // events emitted while the backfill is gated + wantLive string // the only emitted event that survives dedup + }{ + { + name: "the dedup bound is the max backfill timestamp, not the last row", + backfill: []string{"2026-01-01T00:00:05Z", "2026-01-01T00:00:03Z"}, // descending + emit: []string{"2026-01-01T00:00:04Z", "2026-01-01T00:00:06Z"}, + wantLive: "2026-01-01T00:00:06Z", + }, + { + name: "an event older than the backfill is dropped on flush", + backfill: []string{"2026-01-01T00:00:01Z"}, + emit: []string{"2026-01-01T00:00:02Z", "2026-01-01T00:00:00Z"}, + wantLive: "2026-01-01T00:00:02Z", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rows := make([]map[string]any, len(tt.backfill)) + for i, ts := range tt.backfill { + rows[i] = map[string]any{"page": "/a", "received_timestamp": ts} + } + gate := make(chan struct{}) + initialCh := make(chan []map[string]any, 1) + nextCh := make(chan StreamEvent, 8) + + sc := bareStream() + lq := newLiveQuery(sc, + func(context.Context) ([]map[string]any, error) { + <-gate + return rows, nil + }, + &StreamSubscriber{ + Initial: func(got []map[string]any, err error) { + if err != nil { + t.Errorf("Initial err: %v", err) + } + initialCh <- got + }, + Next: func(e StreamEvent) { nextCh <- e }, + }) + defer lq.Close() + + for _, ts := range tt.emit { + sc.emitEvent(liveEvent(ts)) + } + close(gate) + + if got := awaitInitial(t, initialCh); len(got) != len(rows) { + t.Fatalf("want %d backfill rows, got %d", len(rows), len(got)) + } + select { + case e := <-nextCh: + if e.Timestamp != tt.wantLive { + t.Fatalf("want %s delivered, got %s", tt.wantLive, e.Timestamp) + } + case <-time.After(5 * time.Second): + t.Fatal("buffered live event never flushed") + } + select { + case e := <-nextCh: + t.Fatalf("stale event delivered despite dedup: %s", e.Timestamp) + case <-time.After(100 * time.Millisecond): + } + }) + } +} + +func TestLiveQuery_FetchErrorReportedOnce(t *testing.T) { + sc := bareStream() + errCh := make(chan error, 1) + + lq := newLiveQuery(sc, + func(context.Context) ([]map[string]any, error) { return nil, errors.New("boom") }, + &StreamSubscriber{ + Initial: func(_ []map[string]any, err error) { errCh <- err }, + }) + defer lq.Close() + + select { + case err := <-errCh: + if err == nil || err.Error() != "boom" { + t.Fatalf("want boom, got %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Initial never fired on fetch error") + } +} + +func TestLiveQuery_NoCallbacksAfterClose(t *testing.T) { + sc := bareStream() + initialCh := make(chan []map[string]any, 1) + var delivered atomic.Int64 + + lq := newLiveQuery(sc, + func(context.Context) ([]map[string]any, error) { return nil, nil }, + &StreamSubscriber{ + Initial: func(rows []map[string]any, _ error) { initialCh <- rows }, + Next: func(StreamEvent) { delivered.Add(1) }, + Status: func(StreamStatus) { delivered.Add(1) }, + Error: func(error) { delivered.Add(1) }, + }) + awaitInitial(t, initialCh) + + lq.Close() + before := delivered.Load() + sc.emitEvent(liveEvent("2026-01-01T00:00:09Z")) + sc.emitError(errors.New("late")) + sc.setStatus(StatusReconnecting) + time.Sleep(50 * time.Millisecond) + if got := delivered.Load(); got != before { + t.Fatalf("callbacks fired after Close: before=%d after=%d", before, got) + } +} diff --git a/clients/go/namespaces_test.go b/clients/go/namespaces_test.go new file mode 100644 index 00000000..12364138 --- /dev/null +++ b/clients/go/namespaces_test.go @@ -0,0 +1,195 @@ +package wavehouse + +import ( + "context" + "testing" +) + +// TestNamespaces_RequestShape: the method, path and query parameters every +// namespace helper puts on the wire, and the shape it decodes the reply into. +func TestNamespaces_RequestShape(t *testing.T) { + ctx := context.Background() + policy := &Policy{Tables: map[string]TablePolicy{}} + + tests := []struct { + name string + reply any // nil replies with a bare 200 + call func(*testing.T, *Client) + wantMethod string + wantPath string + wantTable string // expected ?table= parameter, if any + }{ + { + name: "Sys.Health", + call: func(t *testing.T, c *Client) { mustNoErr(t, c.Sys.Health(ctx)) }, + wantMethod: "GET", + wantPath: "/v1/health", + }, + { + name: "Schema.List keys tables by name", + reply: []TableSchema{{Name: "clicks", Columns: []Column{{Name: "page", Type: "String"}}}}, + call: func(t *testing.T, c *Client) { + schemas, err := c.Schema.List(ctx) + if err != nil { + t.Fatal(err) + } + if _, ok := schemas["clicks"]; !ok { + t.Fatalf("want clicks in schemas, got %v", schemas) + } + }, + wantMethod: "GET", + wantPath: "/v1/ops/schema", + }, + { + name: "Schema.Refresh", + call: func(t *testing.T, c *Client) { mustNoErr(t, c.Schema.Refresh(ctx)) }, + wantMethod: "POST", + wantPath: "/v1/ops/schema/refresh", + }, + { + name: "Policy.Get", + reply: policy, + call: func(t *testing.T, c *Client) { + pol, err := c.Policy.Get(ctx) + if err != nil { + t.Fatal(err) + } + if pol.Tables == nil { + t.Fatal("want a tables map") + } + }, + wantMethod: "GET", + wantPath: "/v1/ops/policy", + }, + { + name: "Policy.Set", + call: func(t *testing.T, c *Client) { mustNoErr(t, c.Policy.Set(ctx, policy)) }, + wantMethod: "PUT", + wantPath: "/v1/ops/policy", + }, + { + name: "Policy.Validate", + reply: ValidationResult{Valid: true}, + call: func(t *testing.T, c *Client) { + v, err := c.Policy.Validate(ctx, policy) + if err != nil { + t.Fatal(err) + } + if !v.Valid { + t.Fatal("want valid=true") + } + }, + wantMethod: "POST", + wantPath: "/v1/ops/policy/validate", + }, + { + name: "DLQ.List totals every table", + reply: DLQStats{Tables: map[string]int{"clicks": 3}, Total: 3}, + call: func(t *testing.T, c *Client) { + stats, err := c.DLQ.List(ctx) + if err != nil { + t.Fatal(err) + } + if stats.Total != 3 { + t.Fatalf("want total=3, got %d", stats.Total) + } + }, + wantMethod: "GET", + wantPath: "/v1/ops/dlq/stats", + }, + { + name: "DLQ.Table filters by table", + reply: DLQStats{Tables: map[string]int{"clicks": 2}, Total: 2}, + call: func(t *testing.T, c *Client) { + _, err := c.DLQ.Table(ctx, "clicks") + mustNoErr(t, err) + }, + wantMethod: "GET", + wantPath: "/v1/ops/dlq/stats", + wantTable: "clicks", + }, + { + name: "Pipes.List", + reply: []Pipe{{Name: "p1", SQL: "SELECT 1"}}, + call: func(t *testing.T, c *Client) { + pipes, err := c.Pipes.List(ctx) + if err != nil { + t.Fatal(err) + } + if len(pipes) != 1 || pipes[0].Name != "p1" { + t.Fatalf("want [p1], got %v", pipes) + } + }, + wantMethod: "GET", + wantPath: "/v1/ops/pipes", + }, + { + name: "Pipes.Get", + reply: Pipe{Name: "p1", SQL: "SELECT 1"}, + call: func(t *testing.T, c *Client) { + p, err := c.Pipes.Get(ctx, "p1") + if err != nil { + t.Fatal(err) + } + if p.Name != "p1" { + t.Fatalf("want p1, got %s", p.Name) + } + }, + wantMethod: "GET", + wantPath: "/v1/ops/pipes/p1", + }, + { + name: "Pipes.Set", + call: func(t *testing.T, c *Client) { mustNoErr(t, c.Pipes.Set(ctx, "p1", PipeDef{SQL: "SELECT 1"})) }, + wantMethod: "PUT", + wantPath: "/v1/ops/pipes/p1", + }, + { + name: "Pipes.Delete", + call: func(t *testing.T, c *Client) { mustNoErr(t, c.Pipes.Delete(ctx, "p1")) }, + wantMethod: "DELETE", + wantPath: "/v1/ops/pipes/p1", + }, + { + name: "PipeRef.Fetch posts the pipe's parameters", + reply: []map[string]any{{"count": 42}}, + call: func(t *testing.T, c *Client) { + rows, err := Fetch[map[string]any](ctx, c.Pipe("top_pages", map[string]any{"limit": 10})) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 { + t.Fatalf("want 1 row, got %d", len(rows)) + } + }, + wantMethod: "POST", + wantPath: "/v1/pipes/top_pages", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + respond := ok200 + if tt.reply != nil { + respond = jsonResponse(tt.reply) + } + c, reqs := recordingClient(t, respond) + tt.call(t, c) + + got := <-reqs + if got.method != tt.wantMethod || got.path != tt.wantPath { + t.Fatalf("want %s %s, got %s %s", tt.wantMethod, tt.wantPath, got.method, got.path) + } + if tbl := got.query.Get("table"); tbl != tt.wantTable { + t.Fatalf("want table=%q, got %q", tt.wantTable, tbl) + } + }) + } +} + +func mustNoErr(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} diff --git a/clients/go/pipes.go b/clients/go/pipes.go new file mode 100644 index 00000000..d97bfe43 --- /dev/null +++ b/clients/go/pipes.go @@ -0,0 +1,106 @@ +package wavehouse + +import ( + "context" + "fmt" + "net/url" +) + +// PipesNamespace provides admin-only named-pipe management. +type PipesNamespace struct { + ctx httpContext +} + +// List returns all registered pipes. Admin-only. +func (p *PipesNamespace) List(ctx context.Context) ([]Pipe, error) { + var pipes []Pipe + if err := doRequest(ctx, p.ctx, requestOptions{ + method: "GET", + path: "/v1/ops/pipes", + }, &pipes); err != nil { + return nil, fmt.Errorf("list pipes: %w", err) + } + return pipes, nil +} + +// Get returns a single pipe definition by name. Admin-only. +func (p *PipesNamespace) Get(ctx context.Context, name string) (*Pipe, error) { + var pipe Pipe + if err := doRequest(ctx, p.ctx, requestOptions{ + method: "GET", + path: "/v1/ops/pipes/" + url.PathEscape(name), + }, &pipe); err != nil { + return nil, fmt.Errorf("get pipe %q: %w", name, err) + } + return &pipe, nil +} + +// Set creates or updates a pipe. Admin-only. +func (p *PipesNamespace) Set(ctx context.Context, name string, def PipeDef) error { + if err := doRequest(ctx, p.ctx, requestOptions{ + method: "PUT", + path: "/v1/ops/pipes/" + url.PathEscape(name), + body: def, + }, nil); err != nil { + return fmt.Errorf("set pipe %q: %w", name, err) + } + return nil +} + +// Delete removes a pipe by name. Admin-only. +func (p *PipesNamespace) Delete(ctx context.Context, name string) error { + if err := doRequest(ctx, p.ctx, requestOptions{ + method: "DELETE", + path: "/v1/ops/pipes/" + url.PathEscape(name), + }, nil); err != nil { + return fmt.Errorf("delete pipe %q: %w", name, err) + } + return nil +} + +// PipeDef is the definition body for creating/updating a pipe (Pipe minus name). +type PipeDef struct { + SQL string `json:"sql"` + Parameters []ParamDef `json:"parameters,omitempty"` + Description string `json:"description,omitempty"` + AllowedRoles []string `json:"allowed_roles,omitempty"` +} + +// PipeRef is a reference to a named query pipe. Use Fetch to execute it. +type PipeRef struct { + ctx httpContext + name string + params map[string]any + createStream func(table string, opts *StreamOptions) *StreamController +} + +// Fetch executes the pipe and returns the result rows decoded into []T. +func Fetch[Row any](ctx context.Context, p *PipeRef) ([]Row, error) { + body := p.params + if body == nil { + body = map[string]any{} + } + var rows []Row + if err := doRequest(ctx, p.ctx, requestOptions{ + method: "POST", + path: "/v1/pipes/" + url.PathEscape(p.name), + body: body, + }, &rows); err != nil { + return nil, fmt.Errorf("execute pipe %q: %w", p.name, err) + } + return rows, nil +} + +// FetchUntyped executes the pipe and returns rows as []map[string]any. +func (p *PipeRef) FetchUntyped(ctx context.Context) ([]map[string]any, error) { + return Fetch[map[string]any](ctx, p) +} + +// Stream subscribes to live events using the pipe's name as a table name. The +// pipe's SQL and params are NOT applied: where a table of that name exists the +// caller receives its raw events, and otherwise the stream stays silent. Kept +// for parity with the TypeScript SDK's PipeRef.stream(), which behaves the same +// way; both wait on a pipe-aware stream endpoint (issue #445). +func (p *PipeRef) Stream(opts *StreamOptions) *StreamController { + return p.createStream(p.name, opts) +} diff --git a/clients/go/policy.go b/clients/go/policy.go new file mode 100644 index 00000000..7e356bee --- /dev/null +++ b/clients/go/policy.go @@ -0,0 +1,48 @@ +package wavehouse + +import ( + "context" + "fmt" +) + +// PolicyNamespace provides admin-only access-control policy management. +type PolicyNamespace struct { + ctx httpContext +} + +// Get returns the current access-control policy. Admin-only. +func (p *PolicyNamespace) Get(ctx context.Context) (*Policy, error) { + var pol Policy + if err := doRequest(ctx, p.ctx, requestOptions{ + method: "GET", + path: "/v1/ops/policy", + }, &pol); err != nil { + return nil, fmt.Errorf("get policy: %w", err) + } + return &pol, nil +} + +// Set replaces the entire access-control policy. Admin-only. +func (p *PolicyNamespace) Set(ctx context.Context, pol *Policy) error { + if err := doRequest(ctx, p.ctx, requestOptions{ + method: "PUT", + path: "/v1/ops/policy", + body: pol, + }, nil); err != nil { + return fmt.Errorf("set policy: %w", err) + } + return nil +} + +// Validate checks a policy without applying it (dry run). Admin-only. +func (p *PolicyNamespace) Validate(ctx context.Context, pol *Policy) (*ValidationResult, error) { + var result ValidationResult + if err := doRequest(ctx, p.ctx, requestOptions{ + method: "POST", + path: "/v1/ops/policy/validate", + body: pol, + }, &result); err != nil { + return nil, fmt.Errorf("validate policy: %w", err) + } + return &result, nil +} diff --git a/clients/go/query_builder.go b/clients/go/query_builder.go new file mode 100644 index 00000000..dc9e83bc --- /dev/null +++ b/clients/go/query_builder.go @@ -0,0 +1,310 @@ +package wavehouse + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/url" +) + +// DefaultLimit is applied when no explicit limit is set. It is deliberately +// tighter than the backend's DefaultMaxRows (10000) safety cap. +const DefaultLimit = 1000 + +// queryState is the immutable core of a QueryBuilder. +type queryState struct { + table string + columns []string + selectAll bool + aggregations []Aggregation + filters []QueryFilter + groupBy []string + orderBy []OrderClause + limit *int + timeRange *TimeRange + cacheTTL *int // client-side only, not sent to server (#280) +} + +// QueryBuilder builds structured queries. It is immutable: every chain method +// returns a new builder. Use [FetchTyped] or FetchUntyped to execute. +type QueryBuilder struct { + ctx httpContext + createStream func(table string, opts *StreamOptions) *StreamController + state queryState +} + +func (q *QueryBuilder) clone(mutate func(*queryState)) *QueryBuilder { + s := q.state + // Deep-copy slices so mutations don't alias. + s.columns = append([]string(nil), s.columns...) + s.aggregations = append([]Aggregation(nil), s.aggregations...) + s.filters = append([]QueryFilter(nil), s.filters...) + s.groupBy = append([]string(nil), s.groupBy...) + s.orderBy = append([]OrderClause(nil), s.orderBy...) + mutate(&s) + return &QueryBuilder{ctx: q.ctx, createStream: q.createStream, state: s} +} + +// Select appends columns to the projection. +func (q *QueryBuilder) Select(columns ...string) *QueryBuilder { + return q.clone(func(s *queryState) { + s.columns = append(s.columns, columns...) + }) +} + +// SelectAll requests every column the caller's role may read. +func (q *QueryBuilder) SelectAll() *QueryBuilder { + return q.clone(func(s *queryState) { + s.selectAll = true + }) +} + +// Where adds a filter condition. +func (q *QueryBuilder) Where(column string, op FilterOp, value any) *QueryBuilder { + wireOp, ok := opMap[op] + if !ok { + wireOp = string(op) + } + return q.clone(func(s *queryState) { + s.filters = append(s.filters, QueryFilter{Column: column, Op: wireOp, Value: value}) + }) +} + +// Count adds a COUNT aggregation. +func (q *QueryBuilder) Count(column, alias string) *QueryBuilder { + if column == "" { + column = "*" + } + if alias == "" { + alias = "count" + } + return q.addAgg("count", column, alias) +} + +// Sum adds a SUM aggregation. +func (q *QueryBuilder) Sum(column, alias string) *QueryBuilder { + return q.aggDefault("sum", "sum_", column, alias) +} + +// Avg adds an AVG aggregation. +func (q *QueryBuilder) Avg(column, alias string) *QueryBuilder { + return q.aggDefault("avg", "avg_", column, alias) +} + +// Min adds a MIN aggregation. +func (q *QueryBuilder) Min(column, alias string) *QueryBuilder { + return q.aggDefault("min", "min_", column, alias) +} + +// Max adds a MAX aggregation. +func (q *QueryBuilder) Max(column, alias string) *QueryBuilder { + return q.aggDefault("max", "max_", column, alias) +} + +// CountDistinct adds a COUNT DISTINCT aggregation. +func (q *QueryBuilder) CountDistinct(column, alias string) *QueryBuilder { + return q.aggDefault("countDistinct", "count_distinct_", column, alias) +} + +// Aggregate adds a custom aggregation function. +func (q *QueryBuilder) Aggregate(fn, column, alias string) *QueryBuilder { + return q.addAgg(fn, column, alias) +} + +// GroupBy appends columns to the GROUP BY clause. +func (q *QueryBuilder) GroupBy(columns ...string) *QueryBuilder { + return q.clone(func(s *queryState) { + s.groupBy = append(s.groupBy, columns...) + }) +} + +// OrderBy appends an ORDER BY clause. dir defaults to "asc". +func (q *QueryBuilder) OrderBy(column, dir string) *QueryBuilder { + if dir == "" { + dir = "asc" + } + return q.clone(func(s *queryState) { + s.orderBy = append(s.orderBy, OrderClause{Column: column, Dir: dir}) + }) +} + +// Limit sets the maximum number of rows to return. +func (q *QueryBuilder) Limit(n int) *QueryBuilder { + return q.clone(func(s *queryState) { + s.limit = &n + }) +} + +// TimeRange filters by a time window. since and until accept RFC3339 timestamps +// or relative durations ("1h", "30m", "7d", "2w"). +func (q *QueryBuilder) TimeRange(column, since, until string) *QueryBuilder { + return q.clone(func(s *queryState) { + s.timeRange = &TimeRange{Column: column, Since: since, Until: until} + }) +} + +// CacheTTL records a desired result-cache TTL. Currently client-side only — +// the server derives TTLs adaptively (#280). +func (q *QueryBuilder) CacheTTL(seconds int) *QueryBuilder { + return q.clone(func(s *queryState) { + s.cacheTTL = &seconds + }) +} + +// FetchTyped executes the query and decodes rows into []T. +func FetchTyped[Row any](ctx context.Context, q *QueryBuilder) (*Page[Row], error) { + limit := DefaultLimit + if q.state.limit != nil { + limit = *q.state.limit + } + ast := q.buildAST(limit) + + var rows []Row + if err := doRequest(ctx, q.ctx, requestOptions{ + method: "POST", + path: "/v1/query", + params: url.Values{"table": {q.state.table}}, + body: ast, + }, &rows); err != nil { + return nil, err + } + + hasMore := limit > 0 && len(rows) >= limit + page := &Page[Row]{Data: rows, HasMore: hasMore} + + // Attached whenever there is an order column to build a cursor from; a + // projection that omits that column yields an empty next page rather than + // an error. + if hasMore && len(q.state.orderBy) > 0 { + page.Next = func(ctx context.Context) (*Page[Row], error) { + return fetchNextTyped(ctx, q, rows) + } + } + + return page, nil +} + +// FetchUntyped executes the query and returns rows as []map[string]any. +func (q *QueryBuilder) FetchUntyped(ctx context.Context) (*Page[map[string]any], error) { + return FetchTyped[map[string]any](ctx, q) +} + +// Stream opens a live SSE event stream for this query's table. +// Filters and column projections are applied client-side. +func (q *QueryBuilder) Stream(opts *StreamOptions) *StreamController { + raw := q.createStream(q.state.table, opts) + if len(q.state.filters) == 0 && len(q.state.columns) == 0 { + return raw + } + return newFilteredStreamController(raw, q.state.filters, q.state.columns) +} + +// LiveQuery fetches historical data, then streams live updates: the +// subscriber's Initial fires once, then Next fires per live event. Close the +// returned handle to stop it. +func (q *QueryBuilder) LiveQuery(sub *StreamSubscriber, opts *StreamOptions) *LiveQueryHandle { + stream := q.Stream(opts) + fetchFn := func(ctx context.Context) ([]map[string]any, error) { + page, err := q.FetchUntyped(ctx) + if err != nil { + return nil, err + } + return page.Data, nil + } + return newLiveQuery(stream, fetchFn, sub) +} + +func (q *QueryBuilder) aggDefault(fn, prefix, column, alias string) *QueryBuilder { + if alias == "" { + alias = prefix + column + } + return q.addAgg(fn, column, alias) +} + +func (q *QueryBuilder) addAgg(fn, column, alias string) *QueryBuilder { + return q.clone(func(s *queryState) { + s.aggregations = append(s.aggregations, Aggregation{Fn: fn, Column: column, Alias: alias}) + }) +} + +func (q *QueryBuilder) buildAST(effectiveLimit int) *StructuredQuery { + ast := &StructuredQuery{} + hasColumns := len(q.state.columns) > 0 + hasAggs := len(q.state.aggregations) > 0 + + // Projection: explicit select_all, then explicit columns, else — for a bare + // query with no projection and no aggregations — default to select_all so + // from(t).fetch() returns rows. + switch { + case q.state.selectAll: + ast.SelectAll = true + case hasColumns: + ast.Columns = q.state.columns + case !hasAggs: + ast.SelectAll = true + } + + // Empty slices and a nil TimeRange are omitempty-elided, so these can be + // assigned unconditionally. + ast.Aggregations = q.state.aggregations + ast.Filters = q.state.filters + ast.GroupBy = q.state.groupBy + ast.OrderBy = q.state.orderBy + ast.Limit = &effectiveLimit + ast.TimeRange = q.state.timeRange + return ast +} + +func fetchNextTyped[Row any](ctx context.Context, q *QueryBuilder, prevRows []Row) (*Page[Row], error) { + if len(q.state.orderBy) == 0 { + return &Page[Row]{}, nil + } + cursor := q.state.orderBy[0] + + lastRow := any(prevRows[len(prevRows)-1]) + m, ok := lastRow.(map[string]any) + if !ok { + // UseNumber keeps typed int64 cursor values exact past 2^53; rows that + // arrived through the untyped path were already float64 by then. + raw, err := json.Marshal(lastRow) + if err != nil { + // Surfaced rather than swallowed: an empty page here would be + // indistinguishable from normal end-of-pagination. + return nil, fmt.Errorf("wavehouse: marshal cursor row: %w", err) + } + m = make(map[string]any) + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + // Ignored deliberately: a Row that marshals to a non-object leaves m + // empty and ends pagination below, as an absent cursor column does (#452). + _ = dec.Decode(&m) + } + lastValue, exists := m[cursor.Column] + if !exists { + // No cursor value to page from; end pagination quietly. + return &Page[Row]{}, nil + } + + cursorOp := "gt" + if cursor.Dir == "desc" { + cursorOp = "lt" + } + + next := q.clone(func(s *queryState) { + // Replace an existing cursor filter rather than appending, or page N + // would carry N stacked filters on the cursor column. + for i := range s.filters { + if s.filters[i].Column == cursor.Column && s.filters[i].Op == cursorOp { + s.filters[i].Value = lastValue + return + } + } + s.filters = append(s.filters, QueryFilter{ + Column: cursor.Column, + Op: cursorOp, + Value: lastValue, + }) + }) + return FetchTyped[Row](ctx, next) +} diff --git a/clients/go/query_builder_test.go b/clients/go/query_builder_test.go new file mode 100644 index 00000000..733ba5ba --- /dev/null +++ b/clients/go/query_builder_test.go @@ -0,0 +1,378 @@ +package wavehouse + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "sync" + "testing" +) + +// oneRow answers any query with a single-row result set. +var oneRow = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode([]map[string]any{{"page": "/home"}}) +}) + +func TestQueryBuilder_Immutability(t *testing.T) { + b1 := queryTestCtx(t, oneRow).From("clicks").Select("page") + if b2 := b1.Where("score", OpGt, 10); b1 == b2 { + t.Fatal("builder should be immutable — chain methods return new instances") + } +} + +// TestQueryBuilder_RequestBody pins the exact StructuredQuery each chain puts +// on the wire. The table name always travels as a query parameter, never in +// the body — and CacheTTL is client-side only, so it appears in neither. +func TestQueryBuilder_RequestBody(t *testing.T) { + tests := []struct { + name string + run func(context.Context, *TableRef) error + want string + }{ + { + name: "the Fetch shortcut selects everything", + run: func(ctx context.Context, tr *TableRef) error { _, err := tr.Fetch(ctx); return err }, + want: `{"select_all":true,"limit":1000}`, + }, + { + name: "Select projects the named columns", + run: func(ctx context.Context, tr *TableRef) error { return runFetch(ctx, tr.Select("page", "button")) }, + want: `{"columns":["page","button"],"limit":1000}`, + }, + { + name: "SelectAll asks for every readable column", + run: func(ctx context.Context, tr *TableRef) error { return runFetch(ctx, tr.SelectAll()) }, + want: `{"select_all":true,"limit":1000}`, + }, + { + name: "a bare query defaults to select_all", + run: func(ctx context.Context, tr *TableRef) error { return runFetch(ctx, tr.Select()) }, + want: `{"select_all":true,"limit":1000}`, + }, + { + name: "an aggregation-only query does not set select_all", + run: func(ctx context.Context, tr *TableRef) error { return runFetch(ctx, tr.Select().Count("*", "n")) }, + want: `{"aggregations":[{"fn":"count","column":"*","alias":"n"}],"limit":1000}`, + }, + { + name: "Where appends a filter", + run: func(ctx context.Context, tr *TableRef) error { + return runFetch(ctx, tr.Select("page").Where("score", OpGt, 10)) + }, + want: `{"columns":["page"],"filters":[{"column":"score","op":"gt","value":10}],"limit":1000}`, + }, + { + name: "every aggregation helper, with and without an explicit alias", + run: func(ctx context.Context, tr *TableRef) error { + return runFetch(ctx, tr.Select().Count("*", "total").Sum("score", "").Avg("score", ""). + Min("score", "").Max("score", "").CountDistinct("page", ""). + Aggregate("uniqExact", "user_id", "unique_users")) + }, + want: `{"aggregations":[{"fn":"count","column":"*","alias":"total"},` + + `{"fn":"sum","column":"score","alias":"sum_score"},` + + `{"fn":"avg","column":"score","alias":"avg_score"},` + + `{"fn":"min","column":"score","alias":"min_score"},` + + `{"fn":"max","column":"score","alias":"max_score"},` + + `{"fn":"countDistinct","column":"page","alias":"count_distinct_page"},` + + `{"fn":"uniqExact","column":"user_id","alias":"unique_users"}],"limit":1000}`, + }, + { + name: "GroupBy", + run: func(ctx context.Context, tr *TableRef) error { return runFetch(ctx, tr.Select("page").GroupBy("page")) }, + want: `{"columns":["page"],"group_by":["page"],"limit":1000}`, + }, + { + name: "OrderBy", + run: func(ctx context.Context, tr *TableRef) error { + return runFetch(ctx, tr.Select("page").OrderBy("page", "desc")) + }, + want: `{"columns":["page"],"order_by":[{"column":"page","dir":"desc"}],"limit":1000}`, + }, + { + name: "Limit overrides the default", + run: func(ctx context.Context, tr *TableRef) error { return runFetch(ctx, tr.Select("page").Limit(50)) }, + want: `{"columns":["page"],"limit":50}`, + }, + { + name: "TimeRange omits an empty until", + run: func(ctx context.Context, tr *TableRef) error { + return runFetch(ctx, tr.Select("page").TimeRange("received_timestamp", "1h", "")) + }, + want: `{"columns":["page"],"limit":1000,"time_range":{"column":"received_timestamp","since":"1h"}}`, + }, + { + name: "a fully chained query composes every clause", + run: func(ctx context.Context, tr *TableRef) error { + return runFetch(ctx, tr.Select("page").Where("score", OpGt, 10).Count("*", "total"). + GroupBy("page").OrderBy("total", "desc").Limit(50). + TimeRange("received_timestamp", "1h", "").CacheTTL(60)) + }, + want: `{"columns":["page"],"aggregations":[{"fn":"count","column":"*","alias":"total"}],` + + `"filters":[{"column":"score","op":"gt","value":10}],"group_by":["page"],` + + `"order_by":[{"column":"total","dir":"desc"}],"limit":50,` + + `"time_range":{"column":"received_timestamp","since":"1h"}}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, reqs := recordingClient(t, oneRow) + if err := tt.run(context.Background(), c.From("clicks")); err != nil { + t.Fatal(err) + } + got := <-reqs + if got.method != "POST" || got.path != "/v1/query" { + t.Fatalf("want POST /v1/query, got %s %s", got.method, got.path) + } + if tbl := got.query.Get("table"); tbl != "clicks" { + t.Fatalf("want table=clicks in the query string, got %q", tbl) + } + if string(got.body) != tt.want { + t.Fatalf("body:\n got %s\nwant %s", got.body, tt.want) + } + }) + } +} + +// runFetch runs the query and discards the page — the request body is what these +// tests assert on. +func runFetch(ctx context.Context, q *QueryBuilder) error { + _, err := q.FetchUntyped(ctx) + return err +} + +// TestQueryBuilder_AllOperators pins each SDK operator's wire spelling. +func TestQueryBuilder_AllOperators(t *testing.T) { + ops := []struct { + sdk FilterOp + wire string + }{ + {OpEq, "eq"}, + {OpNeq, "neq"}, + {OpGt, "gt"}, + {OpGte, "gte"}, + {OpLt, "lt"}, + {OpLte, "lte"}, + {OpIn, "in"}, + {OpLike, "like"}, + {OpNotLike, "not_like"}, + } + for _, tt := range ops { + t.Run(tt.wire, func(t *testing.T) { + c, reqs := recordingClient(t, oneRow) + if err := runFetch(context.Background(), c.From("clicks").Select("x").Where("col", tt.sdk, "v")); err != nil { + t.Fatal(err) + } + want := fmt.Sprintf(`{"columns":["x"],"filters":[{"column":"col","op":%q,"value":"v"}],"limit":1000}`, tt.wire) + if got := string((<-reqs).body); got != want { + t.Fatalf("want %s, got %s", want, got) + } + }) + } +} + +// pagingServer answers each request with the next fixture page (an empty page +// once they run out) and records every request, so tests can walk page.Next +// and inspect the cursor filters sent. +func pagingServer(t *testing.T, pages [][]map[string]any) (*Client, <-chan recordedRequest) { + t.Helper() + var mu sync.Mutex + call := 0 + return recordingClient(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + idx := call + call++ + mu.Unlock() + page := []map[string]any{} + if idx < len(pages) { + page = pages[idx] + } + _ = json.NewEncoder(w).Encode(page) + })) +} + +func filtersOf(t *testing.T, req recordedRequest) []map[string]any { + t.Helper() + raw, _ := req.jsonBody(t)["filters"].([]any) + out := make([]map[string]any, len(raw)) + for i, f := range raw { + out[i] = f.(map[string]any) + } + return out +} + +// A full page always means HasMore, but Next only exists when there is an +// order column to build a cursor from. The ordered half is covered by +// TestQueryBuilder_PaginationCursor, which walks the cursor it hands back. +func TestQueryBuilder_Pagination_NoOrderNoNext(t *testing.T) { + c, _ := pagingServer(t, [][]map[string]any{{{"id": "a"}, {"id": "b"}}}) + page, err := c.From("clicks").Select("id").Limit(2).FetchUntyped(context.Background()) + if err != nil { + t.Fatal(err) + } + if !page.HasMore { + t.Fatal("a full page means hasMore=true") + } + if page.Next != nil { + t.Fatal("want nil next — no order column for cursor") + } +} + +// TestQueryBuilder_PaginationCursor: every follow-up request carries exactly +// ONE cursor filter — replaced, never stacked — holding the previous page's +// last value, with the operator implied by the sort direction. +func TestQueryBuilder_PaginationCursor(t *testing.T) { + tests := []struct { + name string + dir string + op string + pages [][]map[string]any + cursors []string // expected cursor value per follow-up request + }{ + { + name: "ascending walks forward with gt", dir: "asc", op: "gt", + pages: [][]map[string]any{{{"id": "a"}, {"id": "b"}}, {{"id": "c"}, {"id": "d"}}, {{"id": "e"}}}, + cursors: []string{"b", "d"}, + }, + { + name: "descending walks back with lt", dir: "desc", op: "lt", + pages: [][]map[string]any{{{"id": "z"}, {"id": "y"}}, {{"id": "x"}}}, + cursors: []string{"y"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, reqs := pagingServer(t, tt.pages) + page, err := c.From("clicks").Select("id").OrderBy("id", tt.dir).Limit(2).FetchUntyped(context.Background()) + if err != nil { + t.Fatal(err) + } + if f := filtersOf(t, <-reqs); len(f) != 0 { + t.Fatalf("page 1 must have no cursor filter, got %v", f) + } + for i, want := range tt.cursors { + if page, err = page.Next(context.Background()); err != nil { + t.Fatal(err) + } + if got, want := page.Data[0]["id"], tt.pages[i+1][0]["id"]; got != want { + t.Fatalf("page %d: want first row %v, got %v", i+2, want, got) + } + f := filtersOf(t, <-reqs) + if len(f) != 1 { + t.Fatalf("page %d: want exactly 1 cursor filter, got %v", i+2, f) + } + if f[0]["column"] != "id" || f[0]["op"] != tt.op || f[0]["value"] != want { + t.Fatalf("page %d: unexpected cursor filter %v", i+2, f[0]) + } + } + if page.HasMore { + t.Fatal("the last fixture page is short — want hasMore=false") + } + if n := len(reqs); n != 0 { + t.Fatalf("want no requests beyond the pages walked, got %d more", n) + } + }) + } +} + +func TestQueryBuilder_Pagination_CursorColumnMissingEndsQuietly(t *testing.T) { + c, _ := pagingServer(t, [][]map[string]any{ + {{"other": "1"}, {"other": "2"}}, // projection omits the order column + }) + + page, err := c.From("clicks").Select("other").OrderBy("id", "asc").Limit(2).FetchUntyped(context.Background()) + if err != nil { + t.Fatal(err) + } + next, err := page.Next(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(next.Data) != 0 || next.HasMore || next.Next != nil { + t.Fatalf("want quiet empty page, got %+v", next) + } +} + +// TestQueryBuilder_PaginationCursorPrecision: the cursor value is read back off +// the decoded row, so how the row decoded decides how much precision survives. +// Typed rows keep int64 exactly; the untyped path decodes to float64 and loses +// everything past 2^53 (the same ceiling the TS SDK's JS numbers have). Use +// FetchTyped with an int64 field past 2^53 — documented in queries.mdx. +func TestQueryBuilder_PaginationCursorPrecision(t *testing.T) { + type idRow struct { + ID int64 `json:"id"` + } + const bigID = int64(9007199254740993) // 2^53 + 1: a float64 round-trip corrupts it + + tests := []struct { + name string + walk func(context.Context, *QueryBuilder) error + want string + }{ + { + name: "typed rows keep int64 exactly", + walk: func(ctx context.Context, q *QueryBuilder) error { + page, err := FetchTyped[idRow](ctx, q) + if err != nil { + return err + } + _, err = page.Next(ctx) + return err + }, + want: "9007199254740993", + }, + { + name: "untyped rows hit the float64 ceiling", + walk: func(ctx context.Context, q *QueryBuilder) error { + page, err := q.FetchUntyped(ctx) + if err != nil { + return err + } + _, err = page.Next(ctx) + return err + }, + want: "9007199254740992", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, reqs := pagingServer(t, [][]map[string]any{{{"id": 1}, {"id": bigID}}, {}}) + q := c.From("clicks").Select("id").OrderBy("id", "asc").Limit(2) + if err := tt.walk(context.Background(), q); err != nil { + t.Fatal(err) + } + <-reqs // page 1 carries no cursor + f := filtersOf(t, <-reqs) + if len(f) != 1 { + t.Fatalf("want 1 cursor filter, got %v", f) + } + if got := fmt.Sprint(f[0]["value"]); got != tt.want { + t.Fatalf("cursor value: want %s, got %s (update docs if intentional)", tt.want, got) + } + }) + } +} + +// The cursor round-trip re-marshals the last row to read its cursor value. A +// Row that unmarshals cleanly but can't be marshaled back (an exported func +// field, absent from the response) must surface an error rather than an empty +// page, which is indistinguishable from real exhaustion. +func TestQueryBuilder_Pagination_UnmarshalableRowErrors(t *testing.T) { + type row struct { + ID string `json:"id"` + Cb func() `json:"cb"` + } + c, _ := pagingServer(t, [][]map[string]any{{{"id": "a"}, {"id": "b"}}}) + page, err := FetchTyped[row](context.Background(), + c.From("clicks").Select("id").OrderBy("id", "asc").Limit(2)) + if err != nil { + t.Fatal(err) + } + if page.Next == nil { + t.Fatal("want a Next cursor") + } + if _, err := page.Next(context.Background()); err == nil { + t.Fatal("want a marshal error, got a silently empty page") + } +} diff --git a/clients/go/schema.go b/clients/go/schema.go new file mode 100644 index 00000000..6ac38fe4 --- /dev/null +++ b/clients/go/schema.go @@ -0,0 +1,38 @@ +package wavehouse + +import ( + "context" + "fmt" +) + +// SchemaNamespace provides admin-only schema introspection. +type SchemaNamespace struct { + ctx httpContext +} + +// List returns all table schemas discovered from ClickHouse. Admin-only. +func (s *SchemaNamespace) List(ctx context.Context) (Schemas, error) { + var raw []TableSchema + if err := doRequest(ctx, s.ctx, requestOptions{ + method: "GET", + path: "/v1/ops/schema", + }, &raw); err != nil { + return nil, fmt.Errorf("list schemas: %w", err) + } + schemas := make(Schemas, len(raw)) + for _, t := range raw { + schemas[t.Name] = t + } + return schemas, nil +} + +// Refresh forces a schema re-discovery from ClickHouse. Admin-only. +func (s *SchemaNamespace) Refresh(ctx context.Context) error { + if err := doRequest(ctx, s.ctx, requestOptions{ + method: "POST", + path: "/v1/ops/schema/refresh", + }, nil); err != nil { + return fmt.Errorf("refresh schema: %w", err) + } + return nil +} diff --git a/clients/go/stream.go b/clients/go/stream.go new file mode 100644 index 00000000..bb6050dc --- /dev/null +++ b/clients/go/stream.go @@ -0,0 +1,714 @@ +package wavehouse + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "log" + "mime" + "net/http" + "net/url" + "reflect" + "regexp" + "strings" + "sync" + "time" +) + +// StreamController manages a live SSE event stream. Use Subscribe for +// callback-based consumption or Events for channel-based consumption. +type StreamController struct { + mu sync.Mutex + status StreamStatus + subscribers []*StreamSubscriber + eventCh chan StreamEvent // single buffered channel for Go-native consumption + dropLogOnce sync.Once + cancel context.CancelFunc + done chan struct{} + closed bool + lastErr error // replayed to subscribers that register after it was emitted +} + +// newController builds a controller whose event channel buffers from +// construction, so events predating the first Events call are not lost. +func newController(status StreamStatus, cancel context.CancelFunc) *StreamController { + return &StreamController{ + status: status, + eventCh: make(chan StreamEvent, 256), + cancel: cancel, + done: make(chan struct{}), + } +} + +// newStreamController opens an SSE connection for the given table. +func newStreamController(hctx httpContext, table string, opts *StreamOptions) *StreamController { + ctx, cancel := context.WithCancel(context.Background()) + sc := newController(StatusConnecting, cancel) + go sc.run(ctx, hctx, table, opts) + return sc +} + +// Status returns the current connection status. +func (sc *StreamController) Status() StreamStatus { + sc.mu.Lock() + defer sc.mu.Unlock() + return sc.status +} + +// Subscribe registers callbacks and returns an unsubscribe function. The +// subscriber's Status callback fires immediately with the current status, and +// its Error callback fires with the last error if one was already emitted — +// otherwise a connection that fails before Subscribe returns would deliver it +// to nobody. +func (sc *StreamController) Subscribe(sub *StreamSubscriber) func() { + sc.mu.Lock() + sc.subscribers = append(sc.subscribers, sub) + currentStatus, missedErr := sc.status, sc.lastErr + sc.mu.Unlock() + + // Benign race: setStatus also calls the subscriber, so a stale status here + // is immediately followed by the correct one. + if sub.Status != nil { + sub.Status(currentStatus) + } + if missedErr != nil && sub.Error != nil { + sub.Error(missedErr) + } + + return func() { + sc.mu.Lock() + defer sc.mu.Unlock() + for i, s := range sc.subscribers { + if s == sub { + sc.subscribers = append(sc.subscribers[:i], sc.subscribers[i+1:]...) + break + } + } + } +} + +// Events returns a read-only channel of stream events, closed when the stream +// closes. A consumer that never reads it fills the buffer and trips a +// one-time drop log. +func (sc *StreamController) Events() <-chan StreamEvent { + return sc.eventCh +} + +// Connected blocks until the stream reaches "live" status or the context +// expires. Returns an error if the stream closes before connecting. +func (sc *StreamController) Connected(ctx context.Context) error { + sc.mu.Lock() + if sc.status == StatusLive { + sc.mu.Unlock() + return nil + } + if sc.status == StatusClosed || sc.closed { + sc.mu.Unlock() + return fmt.Errorf("stream is closed") + } + sc.mu.Unlock() + + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-sc.done: + return fmt.Errorf("stream closed before connecting") + case <-ticker.C: + sc.mu.Lock() + s := sc.status + sc.mu.Unlock() + if s == StatusLive { + return nil + } + if s == StatusClosed { + return fmt.Errorf("stream closed before connecting") + } + } + } +} + +// Close shuts down the stream and releases resources. Non-blocking so it is +// safe to call from subscriber callbacks (which run on the stream goroutine). +func (sc *StreamController) Close() { + sc.mu.Lock() + if sc.closed { + sc.mu.Unlock() + return + } + sc.closed = true + sc.mu.Unlock() + + // Deliberately does not wait on sc.done: callbacks run on the stream + // goroutine, so waiting would deadlock a Close made from a callback. + sc.cancel() +} + +func (sc *StreamController) setStatus(s StreamStatus) { + sc.mu.Lock() + // StatusClosed is terminal: a stale Status callback can land after Close + // and must not resurrect the status. + if s == sc.status || sc.status == StatusClosed { + sc.mu.Unlock() + return + } + sc.status = s + subs := append([]*StreamSubscriber(nil), sc.subscribers...) + sc.mu.Unlock() + + for _, sub := range subs { + if sub.Status != nil { + sub.Status(s) + } + } +} + +// snapshotSubs copies the subscriber list under mu so callbacks run unlocked. +// setStatus keeps its own inline copy so the snapshot shares a critical +// section with the status write. +func (sc *StreamController) snapshotSubs() []*StreamSubscriber { + sc.mu.Lock() + defer sc.mu.Unlock() + return append([]*StreamSubscriber(nil), sc.subscribers...) +} + +func (sc *StreamController) emitEvent(event StreamEvent) { + for _, sub := range sc.snapshotSubs() { + if sub.Next != nil { + sub.Next(event) + } + } + + // Guarded by mu so the send and closeEventCh serialize: a late event can + // never hit a closed channel. + sc.mu.Lock() + defer sc.mu.Unlock() + if sc.closed { + return + } + select { + case sc.eventCh <- event: + default: + sc.dropLogOnce.Do(func() { + log.Printf("[wavehouse] stream event dropped: Events() channel buffer full (further drops not logged)") + }) + } +} + +// emitError records err for late subscribers and delivers it to the current +// ones. Recording and snapshotting share one critical section so a subscriber +// registering concurrently is served by exactly one of the two paths. +func (sc *StreamController) emitError(err error) { + sc.mu.Lock() + sc.lastErr = err + subs := append([]*StreamSubscriber(nil), sc.subscribers...) + sc.mu.Unlock() + for _, sub := range subs { + if sub.Error != nil { + sub.Error(err) + } + } +} + +// closeEventCh marks the controller closed and closes the events channel. It +// must serialize with emitEvent's send via mu. +func (sc *StreamController) closeEventCh() { + sc.mu.Lock() + sc.closed = true + close(sc.eventCh) + sc.mu.Unlock() +} + +// run is the SSE connection loop with reconnect/backoff. +func (sc *StreamController) run(ctx context.Context, hctx httpContext, table string, opts *StreamOptions) { + defer func() { + sc.setStatus(StatusClosed) + sc.closeEventCh() + close(sc.done) + }() + + since := "" + if opts != nil { + since = opts.Since + } + + attempt := 0 + for { + if ctx.Err() != nil { + return + } + + lastID, live, err := sc.connect(ctx, hctx, table, since) + // Persist the last event ID so the next reconnect resumes from it. + if lastID != "" { + since = lastID + } + if ctx.Err() != nil { + return + } + + // Reaching "live" resets the backoff so a long-lived stream doesn't + // inherit a maxed-out delay on its first drop. + if live { + attempt = 0 + } + + if err != nil { + // connect classifies its own failures, so Retryable decides + // whether to reconnect: a non-retryable error is terminal. + var apiErr *Error + if errors.As(err, &apiErr) { + sc.emitError(apiErr) + if !apiErr.Retryable { + return + } + } else { + sc.emitError(sseError(0, "SSE_ERROR", err.Error(), true)) + } + } + + sc.setStatus(StatusReconnecting) + select { + case <-ctx.Done(): + return + case <-time.After(backoff(attempt)): + } + attempt++ + } +} + +// connect reads one SSE connection until it closes, returning the last seen +// event ID, whether the connection ever reached the live state, and any error. +func (sc *StreamController) connect(ctx context.Context, hctx httpContext, table, since string) (string, bool, error) { + u, err := url.Parse(hctx.baseURL + "/v1/stream") + if err != nil { + return "", false, sseError(0, "SSE_CONNECT_ERROR", fmt.Sprintf("invalid baseURL: %v", err), false) + } + // A non-HTTP scheme can never carry SSE, so retrying one just spins. + if u.Scheme != "http" && u.Scheme != "https" { + return "", false, sseError(0, "SSE_CONNECT_ERROR", + fmt.Sprintf("baseURL scheme %q is not http or https", u.Scheme), false) + } + q := u.Query() + q.Set("table", table) + if since != "" { + q.Set("since", since) + } + + // Auth travels in the Authorization header, not ?token= as browser + // EventSource requires. + var authHeader string + if hctx.auth != nil { + token, err := hctx.auth(ctx) + if err != nil { + // Retryable: a token endpoint having a bad minute shouldn't tear + // down a healthy long-lived stream. + return "", false, sseError(0, "SSE_AUTH_ERROR", err.Error(), true) + } + if token != "" { + authHeader = "Bearer " + token + } + } + + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil) + if err != nil { + return "", false, err + } + applyConfiguredHeaders(req.Header, hctx.headers) + req.Header.Set("Accept", "text/event-stream") + req.Header.Set("Cache-Control", "no-cache") + if authHeader != "" { + req.Header.Set("Authorization", authHeader) + } + + client := hctx.httpClient + credentialed := authHeader != "" || len(hctx.headers) > 0 + if credentialed { + // Never follow a redirect while carrying a credential: net/http drops + // Authorization across hosts but forwards custom headers verbatim. + // Copied so a caller-supplied client keeps its own CheckRedirect. + c := *client + c.CheckRedirect = func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + } + client = &c + } + + resp, err := client.Do(req) + if err != nil { + if ctx.Err() != nil { + return "", false, errAborted + } + return "", false, sseError(0, "SSE_NETWORK_ERROR", err.Error(), true) + } + defer func() { _ = resp.Body.Close() }() + + if credentialed && resp.StatusCode >= 300 && resp.StatusCode < 400 { + return "", false, sseError(resp.StatusCode, "SSE_REDIRECT", fmt.Sprintf( + "stream endpoint redirected to %q and the SDK did not follow it; redirects are refused while the request carries a credential", + resp.Header.Get("Location")), false) + } + + if resp.StatusCode != http.StatusOK { + return "", false, parseErrorResponse(resp) + } + + // A 200 that isn't an event stream means an intermediary answered; without + // this the stream would sit in StatusLive delivering nothing. + if ct := resp.Header.Get("Content-Type"); !isEventStream(ct) { + shown := ct + if shown == "" { + shown = "(none)" + } + return "", false, sseError(resp.StatusCode, "SSE_BAD_CONTENT_TYPE", + fmt.Sprintf("expected Content-Type text/event-stream, got %s", shown), false) + } + + sc.setStatus(StatusLive) + + scanner := bufio.NewScanner(resp.Body) + // 16 MiB max line: headroom over the ~1 MiB NATS MaxPayload ceiling on a + // single event envelope. + scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) + var eventID, dataLine string + lastID := since + + for scanner.Scan() { + if ctx.Err() != nil { + return lastID, true, nil + } + + line := scanner.Text() + + if line == "" { + // End of an event frame. + if dataLine != "" { + sc.handleSSEData(dataLine) + // Track last event ID for reconnect gap-fill. + if eventID != "" { + lastID = eventID + } + } + eventID = "" + dataLine = "" + continue + } + + if strings.HasPrefix(line, ":") { // comment: keepalive or connected + continue + } + + if strings.HasPrefix(line, "id:") { + eventID = strings.TrimSpace(strings.TrimPrefix(line, "id:")) + } else if strings.HasPrefix(line, "data:") { + trimmed := strings.TrimSpace(strings.TrimPrefix(line, "data:")) + if dataLine == "" { + dataLine = trimmed + } else { + dataLine = dataLine + "\n" + trimmed + } + } + } + + if scanErr := scanner.Err(); scanErr != nil { + return lastID, true, sseError(0, "SSE_READ_ERROR", scanErr.Error(), true) + } + return lastID, true, nil +} + +// sseError builds a stream [*Error] with the SDK's SSE_* taxonomy. +func sseError(status int, code, msg string, retryable bool) *Error { + return &Error{Status: status, Code: code, Message: msg, Retryable: retryable} +} + +// isEventStream reports whether a Content-Type header names text/event-stream, +// ignoring any parameters (charset, boundary) and case. +func isEventStream(contentType string) bool { + mediaType, _, err := mime.ParseMediaType(contentType) + if err != nil { + return false + } + return mediaType == "text/event-stream" +} + +// sseMessage matches the server's SSE event JSON shape. +type sseMessage struct { + TableName string `json:"table_name"` + ReceivedTimestamp string `json:"received_timestamp"` + Data map[string]any `json:"data"` +} + +func (sc *StreamController) handleSSEData(data string) { + var msg sseMessage + if err := json.Unmarshal([]byte(data), &msg); err != nil { + // Reported through the subscriber rather than the global logger, and + // without the payload, which can carry tenant/PII fields. + sc.emitError(sseError(0, "SSE_PARSE_ERROR", + fmt.Sprintf("malformed SSE message (%d bytes): %v", len(data), err), true)) + return + } + + sc.emitEvent(StreamEvent{Table: msg.TableName, Timestamp: msg.ReceivedTimestamp, Data: msg.Data}) +} + +// newFilteredStreamController wraps a StreamController with client-side +// filtering and column projection. +func newFilteredStreamController(inner *StreamController, filters []QueryFilter, columns []string) *StreamController { + compiled := compileFilters(filters) + ctx, cancel := context.WithCancel(context.Background()) + sc := newController(inner.Status(), cancel) + + go func() { + defer func() { + sc.setStatus(StatusClosed) + // closeEventCh serializes with any in-flight emitEvent on the + // inner goroutine, so the channel never closes under a send. + sc.closeEventCh() + close(sc.done) + }() + + unsub := inner.Subscribe(&StreamSubscriber{ + Next: func(event StreamEvent) { + if !matchesFilters(event.Data, compiled) { + return + } + if len(columns) > 0 { + event.Data = projectColumns(event.Data, columns) + } + sc.emitEvent(event) + }, + Status: func(s StreamStatus) { + sc.setStatus(s) + }, + Error: func(err error) { + sc.emitError(err) + }, + }) + + select { + case <-ctx.Done(): + unsub() + inner.Close() + case <-inner.done: + // Unsubscribe anyway so the closed inner controller doesn't + // retain a reference to this wrapper. + unsub() + } + }() + + return sc +} + +// compiledFilter pairs a filter with its precompiled LIKE regex (nil for +// every other operator, or when the pattern isn't a string / doesn't compile). +type compiledFilter struct { + QueryFilter + re *regexp.Regexp +} + +// compileFilters precompiles LIKE/NOT LIKE patterns once per stream, which a +// controller's immutable filter list makes safe. +func compileFilters(filters []QueryFilter) []compiledFilter { + out := make([]compiledFilter, len(filters)) + for i, f := range filters { + out[i] = compiledFilter{QueryFilter: f} + if f.Op == "like" || f.Op == "not_like" { + if pattern, ok := f.Value.(string); ok { + out[i].re = compileLike(pattern) + } + } + } + return out +} + +// compileLike converts a SQL LIKE pattern to a case-insensitive anchored +// regex, returning nil if the pattern doesn't compile. +func compileLike(pattern string) *regexp.Regexp { + escaped := regexp.QuoteMeta(pattern) + escaped = strings.ReplaceAll(escaped, "%", ".*") + escaped = strings.ReplaceAll(escaped, "_", ".") + re, err := regexp.Compile("(?i)^" + escaped + "$") + if err != nil { + return nil + } + return re +} + +// matchesFilters evaluates all filters against a data row (AND). +func matchesFilters(row map[string]any, filters []compiledFilter) bool { + for _, f := range filters { + if !evaluateFilter(row[f.Column], f.Op, f.Value, f.re) { + return false + } + } + return true +} + +func evaluateFilter(actual any, op string, expected any, re *regexp.Regexp) bool { + switch op { + case "eq": + return equalValues(actual, expected) + case "neq": + return !equalValues(actual, expected) + case "gt": + c, ok := compareOrdered(actual, expected) + return ok && c > 0 + case "gte": + c, ok := compareOrdered(actual, expected) + return ok && c >= 0 + case "lt": + c, ok := compareOrdered(actual, expected) + return ok && c < 0 + case "lte": + c, ok := compareOrdered(actual, expected) + return ok && c <= 0 + case "in": + return evaluateIn(actual, expected) + case "like", "not_like": + aStr, ok := actual.(string) + if !ok || re == nil { + return false + } + return (op == "like") == re.MatchString(aStr) + default: + return false + } +} + +// equalValues compares two values for equality, normalizing numeric types and +// comparing timestamps as instants rather than as text (see asInstant). +func equalValues(a, b any) bool { + // nil only equals nil: otherwise the fmt.Sprint fallback would match a + // missing column against the literal string "". + if a == nil || b == nil { + return a == nil && b == nil + } + if af, aOK := toFloat64(a); aOK { + if bf, bOK := toFloat64(b); bOK { + return af == bf + } + } + if at, aOK := asInstant(a); aOK { + if bt, bOK := asInstant(b); bOK { + return at.Equal(bt) + } + } + // fmt.Sprint is safe for all types (no panic on maps/slices). + return fmt.Sprint(a) == fmt.Sprint(b) +} + +// maxTimeOperandChars mirrors the server's row-filter pre-gate: the longest +// accepted spelling is 35 bytes, so this bounds per-event parse work. +const maxTimeOperandChars = 64 + +// asInstant reports the instant a value denotes, and only for RFC 3339 +// spellings carrying an explicit offset or `Z`. Zone-less spellings are +// rejected on purpose: they name an instant only relative to the column's +// timezone, which a subscriber does not know. +func asInstant(v any) (time.Time, bool) { + s, ok := v.(string) + if !ok || len(s) > maxTimeOperandChars { + return time.Time{}, false + } + // Go's RFC3339Nano accepts a ',' decimal separator per ISO 8601 but + // ClickHouse does not, so reject a spelling the server would refuse. + if strings.ContainsRune(s, ',') { + return time.Time{}, false + } + t, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + return time.Time{}, false + } + return t, true +} + +// evaluateIn reports whether actual is contained in the expected slice. +// Reflection handles []any and typed slices alike. +func evaluateIn(actual, expected any) bool { + rv := reflect.ValueOf(expected) + if rv.Kind() != reflect.Slice { + return false + } + for i := range rv.Len() { + if equalValues(actual, rv.Index(i).Interface()) { + return true + } + } + return false +} + +// order reports the sign of a-b, mirroring Go's comparison operators (so an +// incomparable float pair such as NaN reports equal rather than ordered). +func order[T float64 | string](a, b T) (int, bool) { + switch { + case a < b: + return -1, true + case a > b: + return 1, true + default: + return 0, true + } +} + +// compareOrdered returns (-1, 0, or 1) and true for comparable ordered types, +// or (0, false) when the types cannot be compared. +func compareOrdered(actual, expected any) (int, bool) { + if a, aOK := toFloat64(actual); aOK { + if b, bOK := toFloat64(expected); bOK { + return order(a, b) + } + } + // Timestamps compare chronologically and fail closed when only one side is + // a provable instant, matching the server's row filter. + aTime, aIsTime := asInstant(actual) + bTime, bIsTime := asInstant(expected) + if aIsTime || bIsTime { + if !aIsTime || !bIsTime { + return 0, false + } + return aTime.Compare(bTime), true + } + if aStr, ok := actual.(string); ok { + if bStr, ok := expected.(string); ok { + return order(aStr, bStr) + } + } + return 0, false +} + +func toFloat64(v any) (float64, bool) { + switch n := v.(type) { + case float64: + return n, true + case float32: + return float64(n), true + case json.Number: + f, err := n.Float64() + return f, err == nil + } + // All int/uint widths in two cases (row structs often use the narrow ones). + rv := reflect.ValueOf(v) + switch { + case rv.CanInt(): + return float64(rv.Int()), true + case rv.CanUint(): + return float64(rv.Uint()), true + } + return 0, false +} + +func projectColumns(row map[string]any, columns []string) map[string]any { + result := make(map[string]any, len(columns)) + for _, col := range columns { + if v, ok := row[col]; ok { + result[col] = v + } + } + return result +} diff --git a/clients/go/stream_test.go b/clients/go/stream_test.go new file mode 100644 index 00000000..89fe3f84 --- /dev/null +++ b/clients/go/stream_test.go @@ -0,0 +1,718 @@ +package wavehouse + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// sseSender opens w as a flushed text/event-stream response and returns a +// frame writer — the preamble every SSE handler in this file starts with. The +// writer reports false once the client has gone away. +func sseSender(t *testing.T, w http.ResponseWriter) func(frame string) bool { + fl, ok := w.(http.Flusher) + if !ok { + t.Error("response writer is not a flusher") + return func(string) bool { return false } + } + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + fl.Flush() + return func(frame string) bool { + _, err := io.WriteString(w, frame) + fl.Flush() + return err == nil + } +} + +// sseServer serves the given SSE frames on any request, then holds the +// connection open until the client disconnects. +func sseServer(t *testing.T, frames []string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + send := sseSender(t, w) + for _, f := range frames { + send(f) + } + <-r.Context().Done() + })) + t.Cleanup(srv.Close) + return srv +} + +func sseFrame(ts, page string) string { + return "id: " + ts + "\n" + + `data: {"table_name":"clicks","received_timestamp":"` + ts + `","data":{"page":"` + page + `"}}` + + "\n\n" +} + +func streamClient(t *testing.T, srv *httptest.Server) *Client { + t.Helper() + return NewClient(Config{BaseURL: srv.URL, HTTPClient: srv.Client()}) +} + +// firstAPIError subscribes to sc and returns the first error it reports, +// failing the test if none arrives or it isn't a typed *Error. +func firstAPIError(t *testing.T, sc *StreamController) *Error { + t.Helper() + errCh := make(chan error, 4) + sc.Subscribe(&StreamSubscriber{Error: func(err error) { errCh <- err }}) + select { + case err := <-errCh: + var apiErr *Error + if !errors.As(err, &apiErr) { + t.Fatalf("want *Error, got %T: %v", err, err) + } + return apiErr + case <-time.After(5 * time.Second): + t.Fatal("error never surfaced") + return nil + } +} + +func recvEvent(t *testing.T, ch <-chan StreamEvent) StreamEvent { + t.Helper() + select { + case e := <-ch: + return e + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for stream event") + return StreamEvent{} + } +} + +func TestStream_SubscribeReceivesEvents(t *testing.T) { + srv := sseServer(t, []string{ + sseFrame("2026-01-01T00:00:01Z", "/home"), + sseFrame("2026-01-01T00:00:02Z", "/about"), + }) + stream := streamClient(t, srv).From("clicks").Stream(nil) + defer stream.Close() + + got := make(chan StreamEvent, 8) + stream.Subscribe(&StreamSubscriber{ + Next: func(e StreamEvent) { got <- e }, + }) + + e1 := recvEvent(t, got) + if e1.Table != "clicks" || e1.Data["page"] != "/home" { + t.Fatalf("unexpected first event: %+v", e1) + } + e2 := recvEvent(t, got) + if e2.Data["page"] != "/about" || e2.Timestamp != "2026-01-01T00:00:02Z" { + t.Fatalf("unexpected second event: %+v", e2) + } +} + +func TestStream_EventsChannel(t *testing.T) { + srv := sseServer(t, []string{sseFrame("2026-01-01T00:00:01Z", "/home")}) + stream := streamClient(t, srv).From("clicks").Stream(nil) + + ch := stream.Events() + e := recvEvent(t, ch) + if e.Data["page"] != "/home" { + t.Fatalf("unexpected event: %+v", e) + } + + stream.Close() + select { + case _, open := <-ch: + if open { + // A buffered event may arrive before close; drain once more. + if _, open2 := <-ch; open2 { + t.Fatal("events channel not closed after Close") + } + } + case <-time.After(5 * time.Second): + t.Fatal("events channel never closed after Close") + } +} + +func TestStream_Connected(t *testing.T) { + srv := sseServer(t, nil) + stream := streamClient(t, srv).From("clicks").Stream(nil) + defer stream.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := stream.Connected(ctx); err != nil { + t.Fatalf("Connected: %v", err) + } + if s := stream.Status(); s != StatusLive { + t.Fatalf("want live, got %s", s) + } +} + +func TestStream_FilteredDeliversMatchesAndProjects(t *testing.T) { + srv := sseServer(t, []string{ + sseFrame("2026-01-01T00:00:01Z", "/home"), + sseFrame("2026-01-01T00:00:02Z", "/miss"), + sseFrame("2026-01-01T00:00:03Z", "/home"), + }) + stream := streamClient(t, srv).From("clicks"). + Select("page"). + Where("page", OpEq, "/home"). + Stream(nil) + defer stream.Close() + + got := make(chan StreamEvent, 8) + stream.Subscribe(&StreamSubscriber{Next: func(e StreamEvent) { got <- e }}) + + for range 2 { + e := recvEvent(t, got) + if e.Data["page"] != "/home" { + t.Fatalf("filter leaked event: %+v", e) + } + if len(e.Data) != 1 { + t.Fatalf("projection kept extra columns: %+v", e.Data) + } + } + select { + case e := <-got: + t.Fatalf("unexpected third event: %+v", e) + case <-time.After(100 * time.Millisecond): + } +} + +// TestStream_FilteredCloseUnderLoad exercises the wrapper-close path while the +// inner stream is still delivering — the send-on-closed-channel regression. +func TestStream_FilteredCloseUnderLoad(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + send := sseSender(t, w) + for i := 0; ; i++ { + select { + case <-r.Context().Done(): + return + default: + } + if !send(sseFrame(fmt.Sprintf("2026-01-01T00:00:%02dZ", i%60), "/home")) { + return + } + } + })) + t.Cleanup(srv.Close) + + stream := streamClient(t, srv).From("clicks"). + Select(). + Where("page", OpEq, "/home"). + Stream(nil) + + var n atomic.Int64 + stream.Subscribe(&StreamSubscriber{Next: func(StreamEvent) { n.Add(1) }}) + // Also exercise the Events() channel feed path during close. + _ = stream.Events() + + deadline := time.Now().Add(5 * time.Second) + for n.Load() < 10 && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if n.Load() == 0 { + t.Fatal("no events delivered before close") + } + stream.Close() // must not panic or race with in-flight emits + + select { + case <-stream.done: + case <-time.After(5 * time.Second): + t.Fatal("filtered stream goroutine never exited") + } +} + +// TestStream_EventsBufferBeforeFirstEventsCall pins TS parity: the channel +// buffers from construction, so events emitted before the first Events() call +// are still delivered once the consumer starts reading. +func TestStream_EventsBufferBeforeFirstEventsCall(t *testing.T) { + sc := &StreamController{eventCh: make(chan StreamEvent, 256)} + sc.emitEvent(StreamEvent{Table: "clicks", Data: map[string]any{"page": "/"}}) + + select { + case e := <-sc.Events(): + if e.Table != "clicks" { + t.Fatalf("want event for clicks, got %+v", e) + } + default: + t.Fatal("event emitted before Events() was not buffered") + } +} + +// --------------------------------------------------------------------------- +// Client-side filter engine +// --------------------------------------------------------------------------- + +// TestEvaluateFilter covers the operator semantics the client-side filter +// engine has to reproduce. The timestamp block exists because the server +// canonicalizes every top-level DateTime value to RFC 3339 UTC before +// publishing (#402), so a payload and a caller's filter constant routinely +// spell the same instant differently; comparing those as text disagrees with +// the server's row filter, which compares DateTime columns chronologically. +func TestEvaluateFilter(t *testing.T) { + // The canonicalized payload value, and the same instant in +02:00 — which + // sorts ABOVE it lexically ("06" > "04") while being chronologically equal. + const canonical = "2026-06-21T04:00:00Z" + const sameInstantOffset = "2026-06-21T06:00:00+02:00" + const oneSecondLater = "2026-06-21T06:00:01+02:00" + + tests := []struct { + name string + actual any + op string + expected any + want bool + }{ + {"EqNumericCrossType", float64(10), "eq", 10, true}, + {"EqString", "a", "eq", "a", true}, + {"EqMismatch", "a", "eq", "b", false}, + {"Neq", "a", "neq", "b", true}, + {"GtTrue", float64(11), "gt", 10, true}, + {"GtFalse", float64(10), "gt", 10, false}, + {"Gte", float64(10), "gte", 10, true}, + {"Lt", float64(9), "lt", 10, true}, + {"LteString", "a", "lte", "b", true}, + {"GtIncomparable", "a", "gt", 10, false}, + // Narrow/unsigned row-struct fields must compare, not silently drop. + {"GtUnsignedOperand", float64(10), "gt", uint32(5), true}, + {"InAnySlice", "b", "in", []any{"a", "b"}, true}, + {"InTypedSlice", float64(2), "in", []int{1, 2}, true}, + {"InMiss", "c", "in", []any{"a", "b"}, false}, + {"InNotASlice", "a", "in", "a", false}, + {"Like", "hello world", "like", "hello%", true}, + {"LikeCaseInsensitive", "HELLO", "like", "hello", true}, + {"LikeUnderscore", "cat", "like", "c_t", true}, + {"LikeAnchored", "xhello", "like", "hello%", false}, + {"NotLike", "abc", "not_like", "x%", true}, + {"LikeNonString", 5, "like", "5", false}, + {"UnknownOp", "a", "regex", "a", false}, + + // Timestamps compare as instants, not as text. + {"TSEqualAcrossOffsets", canonical, "eq", sameInstantOffset, true}, + {"TSNeqFalseAcrossOffsets", canonical, "neq", sameInstantOffset, false}, + {"TSGteAtSameInstant", canonical, "gte", sameInstantOffset, true}, + {"TSLteAtSameInstant", canonical, "lte", sameInstantOffset, true}, + {"TSGtFalseAtSameInstant", canonical, "gt", sameInstantOffset, false}, + {"TSLtSeesLaterOffsetInstant", canonical, "lt", oneSecondLater, true}, + {"TSGtFalseAgainstLaterInstant", canonical, "gt", oneSecondLater, false}, + {"TSInMatchesAcrossOffsets", canonical, "in", []any{"2020-01-01T00:00:00Z", sameInstantOffset}, true}, + + // Same-offset spellings must keep working exactly as before. + {"TSGtWithinUTC", "2026-06-21T04:00:01Z", "gt", canonical, true}, + {"TSLtWithinUTC", "2026-06-21T03:59:59Z", "lt", canonical, true}, + {"TSEqIdenticalText", canonical, "eq", canonical, true}, + {"TSFractionalSecondsOrder", "2026-06-21T04:00:00.500Z", "gt", canonical, true}, + + // A zone-less constant names an instant only relative to the column's + // declared timezone, which a stream subscriber does not have. It must + // not be silently read as UTC — ordering fails closed. + {"TSZonelessFailsClosedOnGt", canonical, "gt", "2026-06-21 03:00:00", false}, + {"TSZonelessFailsClosedOnLt", canonical, "lt", "2026-06-21 05:00:00", false}, + // A ',' fraction is ISO 8601 but not ClickHouse, so it is not an instant. + {"TSCommaFractionIsNotAnInstant", canonical, "eq", "2026-06-21T04:00:00,000Z", false}, + + // nil only equals nil: a column missing from the payload must not match + // the literal string "" through equalValues' fmt.Sprint fallback. + {"NilEqualsNil", nil, "eq", nil, true}, + {"NilIsNotTheStringNil", nil, "eq", "", false}, + {"TheStringNilIsNotNil", "", "eq", nil, false}, + {"NilIsNotEmptyString", nil, "eq", "", false}, + {"NilIsNotZero", nil, "eq", 0, false}, + + // Non-timestamp strings keep lexicographic ordering; numbers are untouched. + {"PlainStringsOrderLexically", "banana", "gt", "apple", true}, + {"PlainStringsCompareEqual", "apple", "eq", "apple", true}, + {"NumbersOrderNumerically", 100.0, "gt", 9.0, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cf := compileFilters([]QueryFilter{{Column: "c", Op: tt.op, Value: tt.expected}})[0] + if got := evaluateFilter(tt.actual, tt.op, tt.expected, cf.re); got != tt.want { + t.Errorf("evaluateFilter(%v, %q, %v) = %v, want %v", tt.actual, tt.op, tt.expected, got, tt.want) + } + }) + } +} + +func TestMatchesFilters_AllMustMatch(t *testing.T) { + row := map[string]any{"page": "/home", "score": float64(10)} + both := []QueryFilter{ + {Column: "page", Op: "eq", Value: "/home"}, + {Column: "score", Op: "gt", Value: 5}, + } + if !matchesFilters(row, compileFilters(both)) { + t.Fatal("want match when every filter passes") + } + oneFails := append(append([]QueryFilter(nil), both...), QueryFilter{Column: "score", Op: "gt", Value: 99}) + if matchesFilters(row, compileFilters(oneFails)) { + t.Fatal("want no match when any filter fails") + } + if !matchesFilters(row, nil) { + t.Fatal("want match with no filters") + } +} + +func TestToFloat64(t *testing.T) { + for _, v := range []any{ + float64(1), float32(1), + int(1), int8(1), int16(1), int32(1), int64(1), + uint(1), uint8(1), uint16(1), uint32(1), uint64(1), + json.Number("1"), + } { + if f, ok := toFloat64(v); !ok || f != 1 { + t.Fatalf("toFloat64(%T) = (%v, %v)", v, f, ok) + } + } + if _, ok := toFloat64("1"); ok { + t.Fatal("strings must not convert") + } +} + +func TestProjectColumns(t *testing.T) { + row := map[string]any{"a": 1, "b": 2, "c": 3} + got := projectColumns(row, []string{"a", "c", "missing"}) + if len(got) != 2 || got["a"] != 1 || got["c"] != 3 { + t.Fatalf("unexpected projection: %+v", got) + } +} + +// A connect that fails before Subscribe returns still has to reach the +// subscriber; without the replay in Subscribe this is a scheduling race that +// only shows up on a loaded machine. +func TestStream_SubscribeReplaysAMissedTerminalError(t *testing.T) { + stream := NewClient(Config{BaseURL: "ws://example.invalid"}).From("clicks").Stream(nil) + defer stream.Close() + <-stream.done // the connect has already failed and emitted + + errCh := make(chan error, 1) + stream.Subscribe(&StreamSubscriber{Error: func(err error) { errCh <- err }}) + select { + case err := <-errCh: + var apiErr *Error + if !errors.As(err, &apiErr) || apiErr.Code != "SSE_CONNECT_ERROR" { + t.Fatalf("want SSE_CONNECT_ERROR, got %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("a subscriber registered after the failure never saw it") + } +} + +// TestStream_TerminalConnectFailures: every way a connection can fail in a way +// reconnecting cannot fix. Each case must surface a specific, non-retryable +// code, close the stream, and leave Connected failing — the generic retryable +// SSE_ERROR would spin here instead. +func TestStream_TerminalConnectFailures(t *testing.T) { + tests := []struct { + name string + handler http.HandlerFunc + baseURL string // overrides the test server URL when non-empty + auth func(context.Context) (string, error) + wantCode string + }{ + { + name: "a 403 the caller's credentials cannot fix", + handler: func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, `{"error":"forbidden"}`, http.StatusForbidden) + }, + wantCode: "HTTP_403", + }, + { + name: "200 that is not an event stream", + handler: func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, "Please sign in") + }, + wantCode: "SSE_BAD_CONTENT_TYPE", + }, + { + name: "200 with no Content-Type at all", + handler: func(w http.ResponseWriter, _ *http.Request) { + w.Header()["Content-Type"] = nil + w.WriteHeader(http.StatusOK) + }, + wantCode: "SSE_BAD_CONTENT_TYPE", + }, + { + name: "credentialed request is redirected", + handler: func(w http.ResponseWriter, _ *http.Request) { + http.Redirect(w, &http.Request{}, "https://elsewhere.example/v1/stream", http.StatusFound) + }, + auth: StaticToken("secret-token"), + wantCode: "SSE_REDIRECT", + }, + { + name: "baseURL scheme cannot carry SSE", + handler: func(http.ResponseWriter, *http.Request) {}, + baseURL: "ws://example.invalid", + wantCode: "SSE_CONNECT_ERROR", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + srv := httptest.NewServer(tc.handler) + t.Cleanup(srv.Close) + + base := srv.URL + if tc.baseURL != "" { + base = tc.baseURL + } + client := NewClient(Config{BaseURL: base, Auth: tc.auth, HTTPClient: srv.Client()}) + + stream := client.From("clicks").Stream(nil) + defer stream.Close() + + apiErr := firstAPIError(t, stream) + if apiErr.Code != tc.wantCode { + t.Fatalf("want code %s, got %s", tc.wantCode, apiErr) + } + if apiErr.Retryable { + t.Fatalf("%s must not be retryable", apiErr.Code) + } + + select { + case <-stream.done: + case <-time.After(5 * time.Second): + t.Fatalf("stream never closed after terminal %s", tc.wantCode) + } + if s := stream.Status(); s != StatusClosed { + t.Fatalf("want closed, got %s", s) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := stream.Connected(ctx); err == nil { + t.Fatal("Connected must fail on a terminally-closed stream") + } + }) + } +} + +// TestStream_ReconnectResumesFromLastEventID: the gap-fill contract. The +// initial request carries StreamOptions.Since; after the connection drops, +// the reconnect carries ?since=. +func TestStream_ReconnectResumesFromLastEventID(t *testing.T) { + var mu sync.Mutex + var sinceParams []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + sinceParams = append(sinceParams, r.URL.Query().Get("since")) + n := len(sinceParams) + mu.Unlock() + + send := sseSender(t, w) + if n == 1 { + send(sseFrame("2026-01-01T00:00:01Z", "/home")) + return // server closes → client must reconnect with since= + } + <-r.Context().Done() + })) + t.Cleanup(srv.Close) + + stream := streamClient(t, srv).From("clicks").Stream(&StreamOptions{Since: "seed-id"}) + defer stream.Close() + + got := make(chan StreamEvent, 4) + stream.Subscribe(&StreamSubscriber{Next: func(e StreamEvent) { got <- e }}) + recvEvent(t, got) + + // Reconnect happens after ~backoff(0) (≈1s with jitter). + deadline := time.Now().Add(10 * time.Second) + for { + mu.Lock() + n := len(sinceParams) + mu.Unlock() + if n >= 2 { + break + } + if time.Now().After(deadline) { + t.Fatal("stream never reconnected") + } + time.Sleep(20 * time.Millisecond) + } + + mu.Lock() + defer mu.Unlock() + if sinceParams[0] != "seed-id" { + t.Fatalf("initial request: want since=seed-id, got %q", sinceParams[0]) + } + if sinceParams[1] != "2026-01-01T00:00:01Z" { + t.Fatalf("reconnect: want since=, got %q", sinceParams[1]) + } +} + +// The SSE transport builds its URL separately from buildURL (stream.go), so a +// BaseURL path prefix needs its own guard. See TestBaseURLPathPrefixIsPreserved. +func TestStreamBaseURLPathPrefixIsPreserved(t *testing.T) { + gotPath := make(chan string, 1) + mux := http.NewServeMux() + mux.HandleFunc("/api/warehouse/v1/stream", func(w http.ResponseWriter, r *http.Request) { + select { + case gotPath <- r.URL.Path: + default: + } + sseSender(t, w) + <-r.Context().Done() + }) + srv := httptest.NewServer(mux) // anything off-prefix 404s + t.Cleanup(srv.Close) + + client := NewClient(Config{ + BaseURL: srv.URL + "/api/warehouse", + Options: &ClientOptions{}, + HTTPClient: srv.Client(), + }) + sc := client.From("clicks").Stream(nil) + t.Cleanup(sc.Close) + + select { + case p := <-gotPath: + if p != "/api/warehouse/v1/stream" { + t.Fatalf("want prefixed stream path, got %q", p) + } + case <-time.After(3 * time.Second): + t.Fatal("stream never reached the prefixed path") + } +} + +// TestStream_RedirectFollowedWhenUncredentialed: the refusal is scoped to +// requests carrying a credential. Without one there is nothing to leak or +// silently downgrade, so the redirect is followed as usual. +func TestStream_RedirectFollowedWhenUncredentialed(t *testing.T) { + target := sseServer(t, []string{sseFrame("2026-01-01T00:00:01Z", "/home")}) + + front := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL+"/v1/stream?table=clicks", http.StatusFound) + })) + t.Cleanup(front.Close) + + stream := streamClient(t, front).From("clicks").Stream(nil) + defer stream.Close() + + events := make(chan StreamEvent, 4) + stream.Subscribe(&StreamSubscriber{Next: func(e StreamEvent) { events <- e }}) + + select { + case e := <-events: + if e.Table != "clicks" { + t.Fatalf("want table clicks, got %s", e.Table) + } + case <-time.After(5 * time.Second): + t.Fatal("redirect was not followed for an uncredentialed stream") + } +} + +// TestStream_MalformedFrameIsTypedAndRetryable: a bad frame must emit no +// event, and must arrive as an *Error so errors.As and IsRetryable work on it +// — a bare fmt.Errorf would leave callers string-matching. +func TestStream_MalformedFrameIsTypedAndRetryable(t *testing.T) { + srv := sseServer(t, []string{"id: 1\ndata: {not json\n\n"}) + stream := streamClient(t, srv).From("clicks").Stream(nil) + defer stream.Close() + + errCh := make(chan error, 4) + events := make(chan StreamEvent, 4) + stream.Subscribe(&StreamSubscriber{ + Next: func(e StreamEvent) { events <- e }, + Error: func(err error) { errCh <- err }, + }) + + select { + case err := <-errCh: + var apiErr *Error + if !errors.As(err, &apiErr) { + t.Fatalf("want *Error, got %T: %v", err, err) + } + if apiErr.Code != "SSE_PARSE_ERROR" || !strings.Contains(apiErr.Message, "malformed SSE message") { + t.Fatalf("want a malformed-SSE parse error, got %s: %v", apiErr.Code, err) + } + if !IsRetryable(err) { + t.Fatal("a malformed frame must stay retryable") + } + if strings.Contains(apiErr.Message, "not json") { + t.Fatal("payload must not be echoed into the error message") + } + case <-time.After(5 * time.Second): + t.Fatal("error never surfaced") + } + // The error is emitted from the same handleSSEData call that would have + // emitted an event, so by now a leaked event would already be queued. + select { + case e := <-events: + t.Fatalf("malformed data emitted an event: %+v", e) + default: + } +} + +// TestStream_ConfiguredHeadersReachTheStream: ClientOptions.Headers apply to +// SSE, not just REST — and the SDK's own headers still win a collision. +func TestStream_ConfiguredHeadersReachTheStream(t *testing.T) { + seen := make(chan http.Header, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case seen <- r.Header.Clone(): + default: + } + sseSender(t, w) + <-r.Context().Done() + })) + t.Cleanup(srv.Close) + + client := NewClient(Config{ + BaseURL: srv.URL, + HTTPClient: srv.Client(), + Options: &ClientOptions{Headers: map[string]string{ + "X-Operator-Key": "op-secret", + "accept": "application/json", // must lose to the SDK's own Accept + }}, + }) + stream := client.From("clicks").Stream(nil) + defer stream.Close() + + select { + case h := <-seen: + if got := h.Get("X-Operator-Key"); got != "op-secret" { + t.Fatalf("want configured header on the stream request, got %q", got) + } + if got := h.Get("Accept"); got != "text/event-stream" { + t.Fatalf("SDK Accept must win, got %q", got) + } + case <-time.After(5 * time.Second): + t.Fatal("stream request never arrived") + } +} + +// TestStream_FilterMatchesCanonicalizedPayload: the end-to-end shape of the +// instant-comparison bug — a caller filters on a non-UTC spelling and the +// server delivers the canonicalized one. +func TestStream_FilterMatchesCanonicalizedPayload(t *testing.T) { + frame := `event: message +id: 2026-06-21T04:00:00Z +data: {"table_name":"clicks","received_timestamp":"2026-06-21T04:00:00Z","data":{"page":"/home","event_ts":"2026-06-21T04:00:00Z"}} + +` + srv := sseServer(t, []string{frame}) + + stream := streamClient(t, srv).From("clicks"). + SelectAll(). + Where("event_ts", OpGte, "2026-06-21T06:00:00+02:00"). + Stream(nil) + defer stream.Close() + + events := make(chan StreamEvent, 4) + stream.Subscribe(&StreamSubscriber{Next: func(e StreamEvent) { events <- e }}) + + select { + case e := <-events: + if e.Data["page"] != "/home" { + t.Fatalf("unexpected row: %v", e.Data) + } + case <-time.After(5 * time.Second): + t.Fatal("a row chronologically equal to the filter constant was withheld") + } +} diff --git a/clients/go/sys.go b/clients/go/sys.go new file mode 100644 index 00000000..64e2ad5c --- /dev/null +++ b/clients/go/sys.go @@ -0,0 +1,23 @@ +package wavehouse + +import ( + "context" + "fmt" +) + +// SysNamespace provides system health checks. +type SysNamespace struct { + ctx httpContext +} + +// Health pings the server's public /v1/health endpoint, returning nil when the +// server is reachable and past boot. +func (s *SysNamespace) Health(ctx context.Context) error { + if err := doRequest(ctx, s.ctx, requestOptions{ + method: "GET", + path: "/v1/health", + }, nil); err != nil { + return fmt.Errorf("health check: %w", err) + } + return nil +} diff --git a/clients/go/table.go b/clients/go/table.go new file mode 100644 index 00000000..edd8e4f4 --- /dev/null +++ b/clients/go/table.go @@ -0,0 +1,187 @@ +package wavehouse + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "reflect" + "strings" +) + +// TableRef is a reference to a table, used for queries, inserts, schema and +// streams. Safe for concurrent use: it holds no mutable state. +type TableRef struct { + ctx httpContext + table string + createStream func(table string, opts *StreamOptions) *StreamController +} + +// Fetch is a SELECT * shortcut with a default limit of 1000. +func (t *TableRef) Fetch(ctx context.Context) (*Page[map[string]any], error) { + return t.SelectAll().Limit(DefaultLimit).FetchUntyped(ctx) +} + +// Select starts building a typed query with the given column projection. +func (t *TableRef) Select(columns ...string) *QueryBuilder { + return &QueryBuilder{ + ctx: t.ctx, + createStream: t.createStream, + state: queryState{ + table: t.table, + columns: columns, + }, + } +} + +// SelectAll starts a query that selects every column the caller's role may read. +func (t *TableRef) SelectAll() *QueryBuilder { + return t.Select().SelectAll() +} + +// Insert inserts one or more rows into this table. A single map or struct is +// sent as JSON; any slice — []map[string]any, a generated/user-defined row +// type such as []ClickRow, etc. — is serialized to NDJSON for batch ingest. +func (t *TableRef) Insert(ctx context.Context, data any) (*InsertResult, error) { + if rows, ok := data.([]map[string]any); ok { + return t.insertBatch(ctx, rows) + } + if rv, ok := sliceValue(data); ok { + return t.insertBatchReflect(ctx, rv) + } + return t.insertSingle(ctx, data) +} + +// sliceValue reports whether data is a slice, returning its reflect.Value for +// iteration. []byte is excluded so it stays an opaque single value, as +// encoding/json treats it, rather than a batch of numbers. +func sliceValue(data any) (reflect.Value, bool) { + if data == nil { + return reflect.Value{}, false + } + if _, isBytes := data.([]byte); isBytes { + return reflect.Value{}, false + } + v := reflect.ValueOf(data) + if v.Kind() != reflect.Slice { + return reflect.Value{}, false + } + return v, true +} + +// InsertNDJSON inserts pre-formatted NDJSON (one record per line). +func (t *TableRef) InsertNDJSON(ctx context.Context, ndjson string) (*InsertResult, error) { + return t.sendNDJSON(ctx, ndjson) +} + +// Schema returns the table's column definitions from ClickHouse. Admin-only. +func (t *TableRef) Schema(ctx context.Context) (*TableSchema, error) { + var schema TableSchema + if err := doRequest(ctx, t.ctx, requestOptions{ + method: "GET", + path: "/v1/ops/schema", + params: url.Values{"table": {t.table}}, + }, &schema); err != nil { + return nil, fmt.Errorf("get schema for table %q: %w", t.table, err) + } + return &schema, nil +} + +// Stream opens a live SSE event stream for this table. +func (t *TableRef) Stream(opts *StreamOptions) *StreamController { + return t.createStream(t.table, opts) +} + +func (t *TableRef) insertSingle(ctx context.Context, data any) (*InsertResult, error) { + var res struct { + OK *bool `json:"ok"` + Duplicate *bool `json:"duplicate"` + } + if err := doRequest(ctx, t.ctx, requestOptions{ + method: "POST", + path: "/v1/ingest", + params: url.Values{"table": {t.table}}, + body: data, + }, &res); err != nil { + return nil, fmt.Errorf("insert into %q: %w", t.table, err) + } + // An absent "ok" field means success. + return &InsertResult{OK: res.OK == nil || *res.OK, Duplicate: res.Duplicate}, nil +} + +func emptyInsertResult() *InsertResult { + // Separate vars, not one aliased &z: the fields are exported *int, so a + // caller writing through one would otherwise mutate all four. + total, succeeded, failed, duplicates := 0, 0, 0, 0 + return &InsertResult{OK: true, Total: &total, Succeeded: &succeeded, Failed: &failed, Duplicates: &duplicates} +} + +func marshalNDJSON(n int, elem func(int) any) (string, error) { + var sb strings.Builder + for i := range n { + if i > 0 { + sb.WriteByte('\n') + } + raw, err := json.Marshal(elem(i)) + if err != nil { + return "", fmt.Errorf("wavehouse: marshal row %d: %w", i, err) + } + sb.Write(raw) + } + return sb.String(), nil +} + +func (t *TableRef) insertBatch(ctx context.Context, rows []map[string]any) (*InsertResult, error) { + if len(rows) == 0 { + return emptyInsertResult(), nil + } + ndjson, err := marshalNDJSON(len(rows), func(i int) any { return rows[i] }) + if err != nil { + return nil, err + } + return t.sendNDJSON(ctx, ndjson) +} + +// insertBatchReflect is the batch path for slice types other than +// []map[string]any. It builds the same NDJSON body as insertBatch so the +// server's per-record summary survives instead of being dropped. +func (t *TableRef) insertBatchReflect(ctx context.Context, rows reflect.Value) (*InsertResult, error) { + if rows.Len() == 0 { + return emptyInsertResult(), nil + } + ndjson, err := marshalNDJSON(rows.Len(), func(i int) any { return rows.Index(i).Interface() }) + if err != nil { + return nil, err + } + return t.sendNDJSON(ctx, ndjson) +} + +func (t *TableRef) sendNDJSON(ctx context.Context, ndjson string) (*InsertResult, error) { + var res struct { + Total int `json:"total"` + Succeeded int `json:"succeeded"` + Failed int `json:"failed"` + Duplicates int `json:"duplicates"` + Results []InsertRecordResult `json:"results"` + } + if err := doRequest(ctx, t.ctx, requestOptions{ + method: "POST", + path: "/v1/ingest", + params: url.Values{"table": {t.table}}, + rawBody: ndjson, + contentType: "application/x-ndjson", + }, &res); err != nil { + return nil, fmt.Errorf("ingest into %q: %w", t.table, err) + } + result := &InsertResult{ + OK: res.Failed == 0, + Total: &res.Total, + Succeeded: &res.Succeeded, + Failed: &res.Failed, + Duplicates: &res.Duplicates, + } + if len(res.Results) > 0 { + result.Results = res.Results + } + return result, nil +} diff --git a/clients/go/table_test.go b/clients/go/table_test.go new file mode 100644 index 00000000..154cfd34 --- /dev/null +++ b/clients/go/table_test.go @@ -0,0 +1,181 @@ +package wavehouse + +import ( + "context" + "net/http" + "testing" +) + +// TestTableRef_Insert: which wire format each argument shape takes on the way +// to /v1/ingest, and how the server's reply maps onto InsertResult. +func TestTableRef_Insert(t *testing.T) { + type ClickRow struct { + Page string `json:"page"` + } + const ndjson = `{"page":"/a"}` + "\n" + `{"page":"/b"}` + + wantOK := func(t *testing.T, r *InsertResult) { + t.Helper() + if !r.OK { + t.Fatalf("want ok=true, got %+v", r) + } + } + batchReply := map[string]any{"total": 2, "succeeded": 2, "failed": 0, "duplicates": 0} + + tests := []struct { + name string + insert func(context.Context, *TableRef) (*InsertResult, error) + reply map[string]any + wantCT string + wantBody string + check func(*testing.T, *InsertResult) + }{ + { + name: "a single map is sent as one JSON value", + insert: func(ctx context.Context, tr *TableRef) (*InsertResult, error) { + return tr.Insert(ctx, map[string]any{"page": "/home"}) + }, + reply: map[string]any{"ok": true}, + wantCT: "application/json", + wantBody: `{"page":"/home"}`, + check: wantOK, + }, + { + name: "a []map batch is sent as NDJSON", + insert: func(ctx context.Context, tr *TableRef) (*InsertResult, error) { + return tr.Insert(ctx, []map[string]any{{"page": "/a"}, {"page": "/b"}}) + }, + reply: batchReply, + wantCT: "application/x-ndjson", + wantBody: ndjson, + check: wantOK, + }, + { + // A typed slice (a generated or user-defined row type such as + // []ClickRow) must take the batch NDJSON path — not fall through to + // insertSingle, which would send the slice as one JSON body and + // silently ignore any per-record failures the server reports. + name: "a typed slice takes the batch path and surfaces per-record failures", + insert: func(ctx context.Context, tr *TableRef) (*InsertResult, error) { + return tr.Insert(ctx, []ClickRow{{Page: "/a"}, {Page: "/b"}}) + }, + reply: map[string]any{"total": 2, "succeeded": 1, "failed": 1, "duplicates": 0}, + wantCT: "application/x-ndjson", + wantBody: ndjson, + check: func(t *testing.T, r *InsertResult) { + t.Helper() + if r.OK { + t.Fatal("want ok=false when a batch record fails") + } + if r.Failed == nil || *r.Failed != 1 { + t.Fatalf("want failed=1, got %v", r.Failed) + } + if r.Total == nil || *r.Total != 2 { + t.Fatalf("want total=2, got %v", r.Total) + } + }, + }, + { + // The batch path posts to the same URL and also yields ok=true, so + // the wire format is the only thing that distinguishes them: one + // opaque JSON value vs. NDJSON of 16 per-byte rows. encoding/json + // base64s a []byte — documented in queries.md as a value the server + // rejects (use InsertNDJSON for raw bytes); pinned here because it + // proves the batch path wasn't taken. + name: "a []byte stays a single opaque value, not 16 rows", + insert: func(ctx context.Context, tr *TableRef) (*InsertResult, error) { + return tr.Insert(ctx, []byte(`{"page":"/home"}`)) + }, + reply: map[string]any{"ok": true}, + wantCT: "application/json", + wantBody: `"eyJwYWdlIjoiL2hvbWUifQ=="`, + check: wantOK, + }, + { + name: "InsertNDJSON forwards pre-formatted lines verbatim", + insert: func(ctx context.Context, tr *TableRef) (*InsertResult, error) { + return tr.InsertNDJSON(ctx, ndjson) + }, + reply: batchReply, + wantCT: "application/x-ndjson", + wantBody: ndjson, + check: func(t *testing.T, r *InsertResult) { + t.Helper() + if r.Total == nil || *r.Total != 2 { + t.Fatalf("want total=2, got %v", r.Total) + } + }, + }, + { + name: "a deduplicated row reports duplicate=true", + insert: func(ctx context.Context, tr *TableRef) (*InsertResult, error) { + return tr.Insert(ctx, map[string]any{"page": "/dup"}) + }, + reply: map[string]any{"duplicate": true}, + wantCT: "application/json", + wantBody: `{"page":"/dup"}`, + check: func(t *testing.T, r *InsertResult) { + t.Helper() + if r.Duplicate == nil || !*r.Duplicate { + t.Fatal("want duplicate=true") + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, reqs := recordingClient(t, jsonResponse(tt.reply)) + result, err := tt.insert(context.Background(), c.From("clicks")) + if err != nil { + t.Fatal(err) + } + got := <-reqs + if got.path != "/v1/ingest" { + t.Fatalf("want /v1/ingest, got %s", got.path) + } + if ct := got.header.Get("Content-Type"); ct != tt.wantCT { + t.Fatalf("want Content-Type %q, got %q", tt.wantCT, ct) + } + if string(got.body) != tt.wantBody { + t.Fatalf("body:\n got %s\nwant %s", got.body, tt.wantBody) + } + tt.check(t, result) + }) + } +} + +func TestTableRef_InsertEmptyBatch(t *testing.T) { + c := queryTestCtx(t, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Error("should not make a request for empty batch") + })) + result, err := c.From("clicks").Insert(context.Background(), []map[string]any{}) + if err != nil { + t.Fatal(err) + } + if !result.OK { + t.Fatal("want ok=true") + } + if result.Total == nil || *result.Total != 0 { + t.Fatal("want total=0") + } +} + +func TestTableRef_Schema(t *testing.T) { + c, reqs := recordingClient(t, jsonResponse(TableSchema{ + Name: "clicks", + Columns: []Column{{Name: "page", Type: "String"}}, + })) + schema, err := c.From("clicks").Schema(context.Background()) + if err != nil { + t.Fatal(err) + } + if got := (<-reqs).query.Get("table"); got != "clicks" { + t.Fatalf("want table=clicks, got %q", got) + } + if schema.Name != "clicks" { + t.Fatalf("want clicks, got %s", schema.Name) + } + if len(schema.Columns) != 1 || schema.Columns[0].Name != "page" { + t.Fatalf("unexpected columns: %v", schema.Columns) + } +} diff --git a/clients/go/testdata/wire_cases.json b/clients/go/testdata/wire_cases.json new file mode 100644 index 00000000..f61efa53 --- /dev/null +++ b/clients/go/testdata/wire_cases.json @@ -0,0 +1,337 @@ +[ + { + "name": "bare query defaults to select_all", + "endpoint": "query", "table": "clicks", "operations": [], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { "select_all": true, "limit": 1000 } + }, + { + "name": "select explicit columns", + "endpoint": "query", "table": "clicks", "operations": [{ "method": "select", "args": ["page", "button"] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { "columns": ["page", "button"], "limit": 1000 } + }, + { + "name": "selectAll sends select_all flag", + "endpoint": "query", "table": "clicks", "operations": [{ "method": "selectAll" }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { "select_all": true, "limit": 1000 } + }, + { + "name": "where with eq operator", + "endpoint": "query", "table": "clicks", + "operations": [{ "method": "select", "args": ["page"] }, { "method": "where", "args": ["page", "=", "/home"] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { + "columns": ["page"], "filters": [{ "column": "page", "op": "eq", "value": "/home" }], "limit": 1000 + } + }, + { + "name": "where with neq operator", + "endpoint": "query", "table": "clicks", + "operations": [{ "method": "select", "args": ["page"] }, { "method": "where", "args": ["page", "!=", "/home"] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { + "columns": ["page"], "filters": [{ "column": "page", "op": "neq", "value": "/home" }], "limit": 1000 + } + }, + { + "name": "where with gt operator", + "endpoint": "query", "table": "clicks", + "operations": [{ "method": "select", "args": ["page"] }, { "method": "where", "args": ["score", ">", 10] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { "columns": ["page"], "filters": [{ "column": "score", "op": "gt", "value": 10 }], "limit": 1000 } + }, + { + "name": "where with gte operator", + "endpoint": "query", "table": "clicks", + "operations": [{ "method": "select", "args": ["page"] }, { "method": "where", "args": ["score", ">=", 10] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { + "columns": ["page"], "filters": [{ "column": "score", "op": "gte", "value": 10 }], "limit": 1000 + } + }, + { + "name": "where with lt operator", + "endpoint": "query", "table": "clicks", + "operations": [{ "method": "select", "args": ["page"] }, { "method": "where", "args": ["score", "<", 5] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { "columns": ["page"], "filters": [{ "column": "score", "op": "lt", "value": 5 }], "limit": 1000 } + }, + { + "name": "where with lte operator", + "endpoint": "query", "table": "clicks", + "operations": [{ "method": "select", "args": ["page"] }, { "method": "where", "args": ["score", "<=", 5] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { "columns": ["page"], "filters": [{ "column": "score", "op": "lte", "value": 5 }], "limit": 1000 } + }, + { + "name": "where with in operator", + "endpoint": "query", "table": "clicks", + "operations": [ + { "method": "select", "args": ["page"] }, + { "method": "where", "args": ["page", "in", ["/home", "/about"]] } + ], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { + "columns": ["page"], "filters": [{ "column": "page", "op": "in", "value": ["/home", "/about"] }], "limit": 1000 + } + }, + { + "name": "where with like operator", + "endpoint": "query", "table": "clicks", + "operations": [{ "method": "select", "args": ["page"] }, { "method": "where", "args": ["page", "like", "/home%"] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { + "columns": ["page"], "filters": [{ "column": "page", "op": "like", "value": "/home%" }], "limit": 1000 + } + }, + { + "name": "count aggregation", + "endpoint": "query", "table": "clicks", "operations": [{ "method": "count", "args": ["*", "total"] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { "aggregations": [{ "fn": "count", "column": "*", "alias": "total" }], "limit": 1000 } + }, + { + "name": "sum aggregation with default alias", + "endpoint": "query", "table": "clicks", "operations": [{ "method": "sum", "args": ["score", ""] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { "aggregations": [{ "fn": "sum", "column": "score", "alias": "sum_score" }], "limit": 1000 } + }, + { + "name": "avg aggregation with default alias", + "endpoint": "query", "table": "clicks", "operations": [{ "method": "avg", "args": ["score", ""] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { "aggregations": [{ "fn": "avg", "column": "score", "alias": "avg_score" }], "limit": 1000 } + }, + { + "name": "min aggregation with default alias", + "endpoint": "query", "table": "clicks", "operations": [{ "method": "min", "args": ["score", ""] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { "aggregations": [{ "fn": "min", "column": "score", "alias": "min_score" }], "limit": 1000 } + }, + { + "name": "max aggregation with default alias", + "endpoint": "query", "table": "clicks", "operations": [{ "method": "max", "args": ["score", ""] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { "aggregations": [{ "fn": "max", "column": "score", "alias": "max_score" }], "limit": 1000 } + }, + { + "name": "countDistinct aggregation with default alias", + "endpoint": "query", "table": "clicks", "operations": [{ "method": "countDistinct", "args": ["page", ""] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { + "aggregations": [{ "fn": "countDistinct", "column": "page", "alias": "count_distinct_page" }], "limit": 1000 + } + }, + { + "name": "custom aggregate function", + "endpoint": "query", "table": "clicks", + "operations": [{ "method": "aggregate", "args": ["uniqExact", "user_id", "unique_users"] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { + "aggregations": [{ "fn": "uniqExact", "column": "user_id", "alias": "unique_users" }], "limit": 1000 + } + }, + { + "name": "groupBy", + "endpoint": "query", "table": "clicks", + "operations": [{ "method": "select", "args": ["page"] }, { "method": "groupBy", "args": ["page"] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { "columns": ["page"], "group_by": ["page"], "limit": 1000 } + }, + { + "name": "orderBy ascending (default)", + "endpoint": "query", "table": "clicks", + "operations": [{ "method": "select", "args": ["page"] }, { "method": "orderBy", "args": ["page", "asc"] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { "columns": ["page"], "order_by": [{ "column": "page", "dir": "asc" }], "limit": 1000 } + }, + { + "name": "orderBy descending", + "endpoint": "query", "table": "clicks", + "operations": [{ "method": "select", "args": ["page"] }, { "method": "orderBy", "args": ["score", "desc"] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { "columns": ["page"], "order_by": [{ "column": "score", "dir": "desc" }], "limit": 1000 } + }, + { + "name": "explicit limit", + "endpoint": "query", "table": "clicks", + "operations": [{ "method": "select", "args": ["page"] }, { "method": "limit", "args": [50] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { "columns": ["page"], "limit": 50 } + }, + { + "name": "timeRange with since only", + "endpoint": "query", "table": "clicks", + "operations": [ + { "method": "select", "args": ["page"] }, + { "method": "timeRange", "args": ["received_timestamp", "1h", ""] } + ], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { + "columns": ["page"], "time_range": { "column": "received_timestamp", "since": "1h" }, "limit": 1000 + } + }, + { + "name": "timeRange with since and until", + "endpoint": "query", "table": "clicks", + "operations": [ + { "method": "select", "args": ["page"] }, + { "method": "timeRange", "args": ["ts", "2026-01-01", "2026-02-01"] } + ], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { + "columns": ["page"], "time_range": { "column": "ts", "since": "2026-01-01", "until": "2026-02-01" }, "limit": 1000 + } + }, + { + "name": "multiple where clauses (AND)", + "endpoint": "query", "table": "clicks", + "operations": [ + { "method": "select", "args": ["page"] }, + { "method": "where", "args": ["score", ">", 10] }, + { "method": "where", "args": ["page", "=", "/home"] } + ], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { + "columns": ["page"], + "filters": [{ "column": "score", "op": "gt", "value": 10 }, { "column": "page", "op": "eq", "value": "/home" }], + "limit": 1000 + } + }, + { + "name": "complex query combining everything", + "endpoint": "query", "table": "clicks", + "operations": [ + { "method": "select", "args": ["page"] }, + { "method": "where", "args": ["score", ">", 10] }, + { "method": "count", "args": ["*", "total"] }, + { "method": "groupBy", "args": ["page"] }, + { "method": "orderBy", "args": ["total", "desc"] }, + { "method": "limit", "args": [50] }, + { "method": "timeRange", "args": ["received_timestamp", "1h", ""] } + ], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { + "columns": ["page"], "filters": [{ "column": "score", "op": "gt", "value": 10 }], + "aggregations": [{ "fn": "count", "column": "*", "alias": "total" }], "group_by": ["page"], + "order_by": [{ "column": "total", "dir": "desc" }], "limit": 50, + "time_range": { "column": "received_timestamp", "since": "1h" } + } + }, + { + "name": "insert single row path", + "endpoint": "ingest", "table": "clicks", + "operations": [{ "method": "insert", "args": [{ "page": "/home", "button": "cta" }] }], + "expected_path": "/v1/ingest?table=clicks", "expected_method": "POST", "expected_content_type": "application/json", + "expected_body": { "page": "/home", "button": "cta" } + }, + { + "name": "insert batch as NDJSON", + "endpoint": "ingest_batch", "table": "clicks", + "operations": [{ "method": "insert", "args": [[{ "page": "/a" }, { "page": "/b" }]] }], + "expected_path": "/v1/ingest?table=clicks", "expected_method": "POST", + "expected_content_type": "application/x-ndjson", "expected_raw_body": "{\"page\":\"/a\"}\n{\"page\":\"/b\"}" + }, + { + "name": "pipe execution", + "endpoint": "pipe", "pipe_name": "top_pages", "pipe_params": { "limit": 10 }, + "expected_path": "/v1/pipes/top_pages", "expected_method": "POST", "expected_body": { "limit": 10 } + }, + { + "name": "pipe execution with no params sends empty object", + "endpoint": "pipe", "pipe_name": "simple", "pipe_params": null, + "expected_path": "/v1/pipes/simple", "expected_method": "POST", "expected_body": {} + }, + { + "name": "raw SQL", + "endpoint": "sql", "sql": "SELECT count() FROM clicks", + "expected_path": "/v1/ops/query", "expected_method": "POST", + "expected_body": { "sql": "SELECT count() FROM clicks" } + }, + { + "name": "health check", + "endpoint": "health", + "expected_path": "/v1/health", "expected_method": "GET" + }, + { + "name": "schema list", + "endpoint": "schema_list", + "expected_path": "/v1/ops/schema", "expected_method": "GET" + }, + { + "name": "schema refresh", + "endpoint": "schema_refresh", + "expected_path": "/v1/ops/schema/refresh", "expected_method": "POST" + }, + { + "name": "policy get", + "endpoint": "policy_get", + "expected_path": "/v1/ops/policy", "expected_method": "GET" + }, + { + "name": "table with special characters URL-encodes correctly", + "endpoint": "query", "table": "my table", + "operations": [{ "method": "select", "args": ["page"] }, { "method": "limit", "args": [10] }], + "expected_path": "/v1/query?table=my+table", "expected_method": "POST", + "expected_body": { "columns": ["page"], "limit": 10 } + }, + { + "name": "DLQ list", + "endpoint": "dlq_list", + "expected_path": "/v1/ops/dlq/stats", "expected_method": "GET" + }, + { + "name": "DLQ table filter", + "endpoint": "dlq_table", "table": "events", + "expected_path": "/v1/ops/dlq/stats?table=events", "expected_method": "GET" + }, + { + "name": "policy set", + "endpoint": "policy_set", "policy_body": { "default_role": "viewer", "tables": { "events": {} } }, + "expected_path": "/v1/ops/policy", "expected_method": "PUT", "expected_content_type": "application/json", + "expected_body": { "default_role": "viewer", "tables": { "events": {} } } + }, + { + "name": "policy validate", + "endpoint": "policy_validate", "policy_body": { "default_role": "viewer", "tables": { "events": {} } }, + "expected_path": "/v1/ops/policy/validate", "expected_method": "POST", "expected_content_type": "application/json", + "expected_body": { "default_role": "viewer", "tables": { "events": {} } } + }, + { + "name": "pipes list", + "endpoint": "pipes_list", + "expected_path": "/v1/ops/pipes", "expected_method": "GET" + }, + { + "name": "pipes get", + "endpoint": "pipes_get", "pipe_name": "my_pipe", + "expected_path": "/v1/ops/pipes/my_pipe", "expected_method": "GET" + }, + { + "name": "pipes set", + "endpoint": "pipes_set", "pipe_name": "my_pipe", + "pipe_def": { + "sql": "SELECT page, count() AS views FROM events GROUP BY page", + "parameters": [{ "name": "limit", "type": "Int32", "default": 100 }], "description": "Top pages by view count" + }, + "expected_path": "/v1/ops/pipes/my_pipe", "expected_method": "PUT", "expected_content_type": "application/json", + "expected_body": { + "sql": "SELECT page, count() AS views FROM events GROUP BY page", + "parameters": [{ "name": "limit", "type": "Int32", "default": 100 }], "description": "Top pages by view count" + } + }, + { + "name": "pipes delete", + "endpoint": "pipes_delete", "pipe_name": "my_pipe", + "expected_path": "/v1/ops/pipes/my_pipe", "expected_method": "DELETE" + }, + { + "name": "cacheTTL is client-side only and not sent on the wire", + "endpoint": "query", "table": "clicks", + "operations": [{ "method": "cacheTTL", "args": [60] }, { "method": "limit", "args": [5] }], + "expected_path": "/v1/query?table=clicks", "expected_method": "POST", + "expected_body": { "select_all": true, "limit": 5 } + } +] diff --git a/clients/go/types.go b/clients/go/types.go new file mode 100644 index 00000000..492eb5e7 --- /dev/null +++ b/clients/go/types.go @@ -0,0 +1,217 @@ +package wavehouse + +import "context" + +// StructuredQuery is the wire format for POST /v1/query. +type StructuredQuery struct { + // Columns names an explicit projection. A literal "*" is a column named + // "*", not a wildcard, and an empty list with no aggregations and no + // SelectAll selects nothing. Mutually exclusive with SelectAll. + Columns []string `json:"columns,omitempty"` + SelectAll bool `json:"select_all,omitempty"` + Aggregations []Aggregation `json:"aggregations,omitempty"` + Filters []QueryFilter `json:"filters,omitempty"` + GroupBy []string `json:"group_by,omitempty"` + OrderBy []OrderClause `json:"order_by,omitempty"` + Limit *int `json:"limit,omitempty"` + TimeRange *TimeRange `json:"time_range,omitempty"` +} + +// Aggregation describes a single aggregation (e.g. count, sum). +type Aggregation struct { + Fn string `json:"fn"` + Column string `json:"column"` + Alias string `json:"alias"` +} + +// QueryFilter describes a single WHERE condition. +type QueryFilter struct { + Column string `json:"column"` + Op string `json:"op"` + Value any `json:"value"` +} + +// OrderClause describes a single ORDER BY clause. +type OrderClause struct { + Column string `json:"column"` + Dir string `json:"dir"` // "asc" or "desc" +} + +// TimeRange filters by a time window on a column. +type TimeRange struct { + Column string `json:"column"` + Since string `json:"since"` + Until string `json:"until,omitempty"` +} + +// FilterOp is an SDK-facing filter operator. +type FilterOp string + +const ( + OpEq FilterOp = "=" + OpNeq FilterOp = "!=" + OpGt FilterOp = ">" + OpGte FilterOp = ">=" + OpLt FilterOp = "<" + OpLte FilterOp = "<=" + OpIn FilterOp = "in" + OpLike FilterOp = "like" + OpNotLike FilterOp = "not_like" +) + +// opMap translates SDK operators to backend wire tokens. +var opMap = map[FilterOp]string{ + OpEq: "eq", + OpNeq: "neq", + OpGt: "gt", + OpGte: "gte", + OpLt: "lt", + OpLte: "lte", + OpIn: "in", + OpLike: "like", + OpNotLike: "not_like", +} + +// Column describes a single column in a table schema. +type Column struct { + Name string `json:"name"` + Type string `json:"type"` + IsNullable bool `json:"is_nullable"` + HasDefault bool `json:"has_default"` +} + +// TableSchema describes a table's schema. +type TableSchema struct { + Name string `json:"name"` + Columns []Column `json:"columns"` +} + +// Schemas maps table names to their schemas. +type Schemas map[string]TableSchema + +// InsertRecordResult is a per-record outcome from a batch insert. +type InsertRecordResult struct { + Index int `json:"index"` + OK *bool `json:"ok,omitempty"` + Duplicate *bool `json:"duplicate,omitempty"` + Error string `json:"error,omitempty"` +} + +// InsertResult is the outcome of an insert operation. +type InsertResult struct { + OK bool `json:"ok"` + Duplicate *bool `json:"duplicate,omitempty"` + Total *int `json:"total,omitempty"` + Succeeded *int `json:"succeeded,omitempty"` + Failed *int `json:"failed,omitempty"` + Duplicates *int `json:"duplicates,omitempty"` + Results []InsertRecordResult `json:"results,omitempty"` +} + +// DLQStats describes dead-letter-queue statistics. +type DLQStats struct { + Tables map[string]int `json:"tables"` + Total int `json:"total"` +} + +// Pipe describes a named query pipe definition. +type Pipe struct { + Name string `json:"name"` + SQL string `json:"sql"` + Parameters []ParamDef `json:"parameters,omitempty"` + Description string `json:"description,omitempty"` + AllowedRoles []string `json:"allowed_roles,omitempty"` +} + +// ParamDef describes a pipe parameter. +type ParamDef struct { + Name string `json:"name"` + Type string `json:"type"` + Required bool `json:"required,omitempty"` + Default any `json:"default,omitempty"` +} + +// Policy describes the server's access-control policy. +type Policy struct { + DefaultRole string `json:"default_role,omitempty"` + // AdminRole grants full access and the allowlist bypass; empty means the + // server's default ("admin"). + AdminRole string `json:"admin_role,omitempty"` + Tables map[string]TablePolicy `json:"tables"` +} + +// TablePolicy describes per-table access control. +type TablePolicy struct { + Select map[string]RolePermissions `json:"select,omitempty"` + Insert map[string]RolePermissions `json:"insert,omitempty"` +} + +// RolePermissions describes a role's access to a table. +type RolePermissions struct { + AllowColumns []string `json:"allow_columns,omitempty"` + DenyColumns []string `json:"deny_columns,omitempty"` + Filter map[string]PolicyFilter `json:"filter,omitempty"` + Check map[string]PolicyFilter `json:"check,omitempty"` + AllowedAggregations []string `json:"allowed_aggregations,omitempty"` + DeniedAggregations []string `json:"denied_aggregations,omitempty"` + MaxRows *int `json:"max_rows,omitempty"` + MaxExecutionTime any `json:"max_execution_time,omitempty"` + MaxRowsToRead *int64 `json:"max_rows_to_read,omitempty"` + MaxMemoryUsage any `json:"max_memory_usage,omitempty"` +} + +// PolicyFilter describes a policy filter predicate. Fields are pointers with +// omitempty so an intentional empty-string comparison is sent as "" while an +// unset operator is omitted entirely, never null. +type PolicyFilter struct { + Eq *string `json:"_eq,omitempty"` + Neq *string `json:"_neq,omitempty"` + Gt *string `json:"_gt,omitempty"` + Lt *string `json:"_lt,omitempty"` + In *string `json:"_in,omitempty"` +} + +// ValidationResult is the response from policy validation. +type ValidationResult struct { + Valid bool `json:"valid"` +} + +// StreamStatus represents the connection state of a stream. +type StreamStatus string + +const ( + StatusConnecting StreamStatus = "connecting" + StatusLive StreamStatus = "live" + StatusReconnecting StreamStatus = "reconnecting" + StatusClosed StreamStatus = "closed" +) + +// StreamEvent is a single event from an SSE stream. +type StreamEvent struct { + Table string `json:"table"` + Timestamp string `json:"timestamp"` + Data map[string]any `json:"data"` +} + +// StreamSubscriber receives events from a stream. Every callback is optional. +type StreamSubscriber struct { + // Initial fires once with the historical backfill (live queries only). + Initial func(rows []map[string]any, err error) + Next func(event StreamEvent) + Status func(status StreamStatus) + Error func(err error) +} + +// StreamOptions configures a stream. +type StreamOptions struct { + // Since is an RFC3339 timestamp for gap-fill replay. + Since string +} + +// Page wraps a result set with pagination metadata. +type Page[T any] struct { + Data []T + HasMore bool + // Next fetches the next page, and is nil when no cursor is available. + Next func(ctx context.Context) (*Page[T], error) +} diff --git a/clients/go/wavehouse.go b/clients/go/wavehouse.go new file mode 100644 index 00000000..ad1b7930 --- /dev/null +++ b/clients/go/wavehouse.go @@ -0,0 +1,153 @@ +// Package wavehouse is the official Go SDK for WaveHouse — a schema-aware +// real-time API gateway for ClickHouse. Zero third-party runtime dependencies. +// +// Create a client with [NewClient], then use [Client.From] for table +// operations, [Client.Pipe] for named queries, or the admin namespaces +// ([Client.Schema], [Client.Policy], etc.) for management. +// +// client := wavehouse.NewClient(wavehouse.Config{ +// BaseURL: "http://localhost:8080", +// }) +// rows, err := client.From("clicks").SelectAll().FetchUntyped(ctx) +package wavehouse + +import ( + "context" + "fmt" + "maps" + "net/http" + "strings" +) + +// defaultMaxRetries is used when [ClientOptions.MaxRetries] is nil. +const defaultMaxRetries = 2 + +// Config configures a [Client]. +type Config struct { + // BaseURL of the WaveHouse server (e.g. "http://localhost:8080"). + BaseURL string + + // Auth returns a bearer token, called once per request. Return "" to skip + // the Authorization header; nil means unauthenticated (server default_role). + Auth func(ctx context.Context) (string, error) + + // Options tunes transport behavior. + Options *ClientOptions + + // HTTPClient overrides the default http.Client. + HTTPClient *http.Client +} + +// ClientOptions tunes transport behavior. +type ClientOptions struct { + // MaxRetries is the maximum number of retry attempts for retryable errors. + // Total attempts = MaxRetries + 1. nil means unset and uses the default of + // 2; use [Ptr] to set it, including Ptr(0) to turn retries off. A pointer + // is required because a plain int could not tell an explicit 0 from a + // caller who set only Headers. Negative values are treated as 0. + MaxRetries *int + + // Headers are sent on every REST call and SSE stream. They are applied + // before the SDK's own headers, so Authorization, Accept, Content-Type and + // Cache-Control win any collision; names are canonicalized by net/http. + Headers map[string]string +} + +// Ptr returns a pointer to v, for the optional fields where nil means "unset" +// and a pointer to the zero value is a deliberate choice — [ClientOptions.MaxRetries] +// and the [PolicyFilter] operators. wavehouse.Ptr(0) disables retries. +func Ptr[T any](v T) *T { return &v } + +// StaticToken returns an Auth function that always returns the same token. +func StaticToken(token string) func(context.Context) (string, error) { + return func(context.Context) (string, error) { return token, nil } +} + +// Client is the WaveHouse SDK entry point. +type Client struct { + ctx httpContext + + Schema *SchemaNamespace + Policy *PolicyNamespace + DLQ *DLQNamespace + Sys *SysNamespace + Pipes *PipesNamespace +} + +// NewClient creates a new WaveHouse client. +func NewClient(cfg Config) *Client { + maxRetries := defaultMaxRetries + if cfg.Options != nil && cfg.Options.MaxRetries != nil { + maxRetries = max(*cfg.Options.MaxRetries, 0) + } + + // Copy so a later mutation of the caller's map can't reach into requests. + var headers map[string]string + if cfg.Options != nil && len(cfg.Options.Headers) > 0 { + headers = maps.Clone(cfg.Options.Headers) + } + + hc := cfg.HTTPClient + if hc == nil { + // Not http.DefaultClient: another package could reconfigure that + // mutable global (timeout, transport, redirects) after we are built. + hc = &http.Client{} + } + + c := &Client{ + ctx: httpContext{ + baseURL: strings.TrimRight(cfg.BaseURL, "/"), + auth: cfg.Auth, + maxRetries: maxRetries, + httpClient: hc, + headers: headers, + }, + } + + c.Schema = &SchemaNamespace{ctx: c.ctx} + c.Policy = &PolicyNamespace{ctx: c.ctx} + c.DLQ = &DLQNamespace{ctx: c.ctx, createStream: c.createStream} + c.Sys = &SysNamespace{ctx: c.ctx} + c.Pipes = &PipesNamespace{ctx: c.ctx} + + return c +} + +// From returns a reference to a table for queries, inserts, and streams. +func (c *Client) From(table string) *TableRef { + return &TableRef{ + ctx: c.ctx, + table: table, + createStream: c.createStream, + } +} + +// Pipe returns a reference to a named query pipe. Pass params for the pipe's +// template parameters. +func (c *Client) Pipe(name string, params map[string]any) *PipeRef { + return &PipeRef{ + ctx: c.ctx, + name: name, + params: params, + createStream: c.createStream, + } +} + +// SQL executes a raw SQL query against ClickHouse, decoding results into []Row. +// Requires the admin role; the server proxies the SQL verbatim. +func SQL[Row any](ctx context.Context, c *Client, query string) ([]Row, error) { + var rows []Row + err := doRequest(ctx, c.ctx, requestOptions{ + method: "POST", + path: "/v1/ops/query", + body: map[string]string{"sql": query}, + }, &rows) + if err != nil { + return nil, fmt.Errorf("sql query: %w", err) + } + return rows, nil +} + +func (c *Client) createStream(table string, opts *StreamOptions) *StreamController { + return newStreamController(c.ctx, table, opts) +} diff --git a/clients/ts/README.md b/clients/ts/README.md index 4ec5a9f2..ce03eaea 100644 --- a/clients/ts/README.md +++ b/clients/ts/README.md @@ -8,7 +8,7 @@ TypeScript client for [WaveHouse](https://github.com/Wave-RF/WaveHouse) — sche npm install @wavehouse/sdk ``` -Requires Node 22 or newer — the only line this SDK is tested against; Node 18 and 20 are past upstream end-of-life. Neither browsers nor Node need a polyfill: streaming runs on `fetch`, like the rest of the SDK — see [Runtime support](https://wavehouse.dev/sdk/#runtime-support). +Requires Node 22 or newer — the only line this SDK is tested against; Node 18 and 20 are past upstream end-of-life. Neither browsers nor Node need a polyfill: streaming runs on `fetch`, like the rest of the SDK — see [Runtime support](https://wavehouse.dev/sdk/setup/typescript#runtime-support). This works in any framework that uses a bundler — React, Vue, Svelte, Angular, Astro, SolidJS, or plain Vite — with `import { createClient } from '@wavehouse/sdk'`. @@ -57,7 +57,7 @@ const wh = createClient({ }); ``` -`baseURL` may include a path prefix (`https://app.example.com/api/wavehouse`) when WaveHouse is served under one — see [Serving under a path prefix](https://wavehouse.dev/sdk#serving-under-a-path-prefix). +`baseURL` may include a path prefix (`https://app.example.com/api/wavehouse`) when WaveHouse is served under one — see [Serving under a path prefix](https://wavehouse.dev/sdk/setup/typescript#serving-under-a-path-prefix). ### Query Data @@ -170,7 +170,7 @@ E2E tests live in `tests/e2e/sdk/` (repo root) and exercise the full pipeline th ## API Reference -See the full [SDK API Reference](https://wavehouse.dev/sdk) for detailed documentation of every method, type, and option. +See the full [SDK documentation](https://wavehouse.dev/sdk/setup/typescript) for detailed documentation of every method, type, and option. ## License diff --git a/docs/src/components/Footer.astro b/docs/src/components/Footer.astro index cfeda0be..87d99617 100644 --- a/docs/src/components/Footer.astro +++ b/docs/src/components/Footer.astro @@ -146,7 +146,7 @@ const trademarkSource = pageText(entry.data, entry.body ?? ""); Getting Started Architecture API Reference - TypeScript SDK + Client SDKs
Operations diff --git a/docs/src/config/sidebar.ts b/docs/src/config/sidebar.ts index b3a61a20..94d2513d 100644 --- a/docs/src/config/sidebar.ts +++ b/docs/src/config/sidebar.ts @@ -25,13 +25,26 @@ export const sidebar: StarlightUserConfig["sidebar"] = [ items: [ { label: "API Reference", slug: "api" }, { - // Topic-first SDK pages: when a second SDK language lands, these - // shared usage pages grow code tabs and each - // language gets its own setup/caveats page — the topic URLs never - // churn (decision in PR #313). - label: "TypeScript SDK", + // Topic-first SDK pages: the shared usage pages carry + // per language, and setup — the one genuinely + // per-language part — is grouped under /sdk/setup/, with the sidebar + // mirroring that segment. The topic URLs never churn as languages are + // added (decision in PR #313, executed for Go in PR #434). A third + // language adds one /sdk/setup/ page, a row in the /sdk/setup + // table, a tab per topic, and one entry in the Setup group below; it + // does NOT add a parallel tree, and no language sits at the root + // of /sdk. + label: "SDKs", items: [ { label: "Overview", slug: "sdk" }, + { + label: "Setup", + items: [ + { label: "All SDKs", slug: "sdk/setup" }, + { label: "TypeScript", slug: "sdk/setup/typescript" }, + { label: "Go", slug: "sdk/setup/go" }, + ], + }, { label: "Queries", slug: "sdk/queries" }, { label: "Streaming & Live Queries", slug: "sdk/streaming" }, { label: "Pipes", slug: "sdk/pipes" }, diff --git a/docs/src/content/docs/404.md b/docs/src/content/docs/404.md index ed74af3a..4eec97c6 100644 --- a/docs/src/content/docs/404.md +++ b/docs/src/content/docs/404.md @@ -45,7 +45,7 @@ head: Getting startedInstall, run, and ingest your first event ArchitectureHow the pieces fit together API referenceEndpoints, payloads, and error semantics - TypeScript SDKTyped client for browser and Node + Client SDKsTyped TypeScript and Go clients

Followed a link that should have worked? File an issue — broken links are bugs.

diff --git a/docs/src/content/docs/access-control.mdx b/docs/src/content/docs/access-control.mdx index 56564f80..f2c323ef 100644 --- a/docs/src/content/docs/access-control.mdx +++ b/docs/src/content/docs/access-control.mdx @@ -184,7 +184,7 @@ The rules, in order: 2. **An empty (or `["*"]`) `allow_columns` means "all columns"** — every column not in `deny_columns` is permitted. Use this with `deny_columns` for a blocklist posture: see everything *except* a few sensitive columns. 3. **A non-empty `allow_columns` is an allowlist** — only the named columns (and never the denied ones) are permitted. -On a structured query (`POST /v1/query?table={table}`) the allowlist is a **hard cap on every column the query references — in any clause**: the projection, an aggregation argument, `filters`, `group_by`, `order_by`, and `time_range`. Naming a disallowed column anywhere is rejected with `403 column "x" not allowed`. A full-row read is requested explicitly with `"select_all": true`, which expands to exactly the columns the role may read — never a raw `SELECT *` that could include a denied column; if the role is allowed *no* columns, the read is rejected (`403`) rather than returning empty rows. **Omitting `columns` (or sending `[]` / `""`) returns nothing** — a request for no data — so a hidden column can't leak by being left out, grouped on, or filtered on to infer its values. (Note: in a query, `["*"]` is the *literal column named `*`*, not a wildcard — use `select_all` for all columns. In `allow_columns`, `["*"]` is still the all-columns wildcard.) On insert (`POST /v1/ingest?table={table}`) the body is `403 column "x" not allowed for insert`. On live streams, denied columns are silently **stripped** from each event rather than rejecting the connection. The structured-query and live-stream paths defer to the **same** per-column decision (`IsColumnAllowed`), so the two read surfaces enforce identical column visibility and can't drift apart. +On a structured query (`POST /v1/query?table={table}`) the allowlist is a **hard cap on every column the query references — in any clause**: the projection, an aggregation argument, `filters`, `group_by`, `order_by`, and `time_range`. Naming a disallowed column anywhere is rejected with `403 column "x" not allowed`. A full-row read is requested explicitly with `"select_all": true`; for a column-restricted role it expands to exactly the columns the role may read rather than a bare `SELECT *` that could include a denied column (an unrestricted/admin role does get `SELECT *`); if the role is allowed *no* columns, the read is rejected (`403`) rather than returning empty rows. **Omitting `columns` (or sending `[]` / `""`) returns nothing** — a request for no data — so a hidden column can't leak by being left out, grouped on, or filtered on to infer its values. (Note: in a query, `["*"]` is the *literal column named `*`*, not a wildcard — use `select_all` for all columns. In `allow_columns`, `["*"]` is still the all-columns wildcard.) On insert (`POST /v1/ingest?table={table}`) the body is `403 column "x" not allowed for insert`. On live streams, denied columns are silently **stripped** from each event rather than rejecting the connection. The structured-query and live-stream paths defer to the **same** per-column decision (`IsColumnAllowed`), so the two read surfaces enforce identical column visibility and can't drift apart. ## Row-level security @@ -451,7 +451,7 @@ curl -X PUT http://localhost:8080/v1/ops/policy \ -d @policy.json ``` -The full request and response shapes for these endpoints live in the [API Reference](/api); the [TypeScript SDK](/sdk) wraps them as `client.policy.get()`, `client.policy.set(policy)`, and `client.policy.validate(policy)`. +The full request and response shapes for these endpoints live in the [API Reference](/api); both [SDKs](/sdk/admin#policy) wrap them — TypeScript as `client.policy.get()`, `client.policy.set(policy)`, and `client.policy.validate(policy)`, Go as `wh.Policy.Get(ctx)`, `wh.Policy.Set(ctx, policy)`, and `wh.Policy.Validate(ctx, policy)`. ## Bootstrapping and the policy lifecycle diff --git a/docs/src/content/docs/api.md b/docs/src/content/docs/api.md index c778a82f..1e44bf72 100644 --- a/docs/src/content/docs/api.md +++ b/docs/src/content/docs/api.md @@ -152,7 +152,7 @@ Status code: `503 Service Unavailable` Returns **`200 OK` with an empty body** once the gateway is past boot, or **`503 Service Unavailable`** (also empty) while boot-time schema discovery is still failing. No authentication required and no response body — the caller only branches on the status code, so there's nothing to JSON-encode or cache per request. -This is what the SDK's `wh.sys.health()` calls, and the endpoint to use when choosing among multiple servers in a distributed setup. It mirrors `/livez` under the hood but is intentionally a `/v1` API route rather than a Kubernetes probe path: an operator may filter the bare probe paths (`/livez`, `/readyz`, `/healthz`) out at the reverse proxy since they're internal probes, so the SDK relies on `/v1/health`, which is documented public API surface meant to stay reachable. It does **not** ping ClickHouse — readiness-based load balancing is the proxy/LB's job (via `/readyz`), not the client's. +This is what the SDKs' liveness ping calls (`wh.sys.health()` / `wh.Sys.Health(ctx)`), and the endpoint to use when choosing among multiple servers in a distributed setup. It mirrors `/livez` under the hood but is intentionally a `/v1` API route rather than a Kubernetes probe path: an operator may filter the bare probe paths (`/livez`, `/readyz`, `/healthz`) out at the reverse proxy since they're internal probes, so the SDKs rely on `/v1/health`, which is documented public API surface meant to stay reachable. It does **not** ping ClickHouse — readiness-based load balancing is the proxy/LB's job (via `/readyz`), not the client's. --- @@ -305,7 +305,7 @@ Examples for a `DateTime64(3, 'America/New_York')` column: `"2026-06-21 00:00:00 #### Batch Ingest -A **JSON array** of objects (`[{…}, {…}]`) or an **NDJSON** body (`Content-Type: application/x-ndjson`, one JSON object per line) ingests a batch in a single request. Each record is validated, authorized, deduplicated, and published independently, so **one malformed or rejected record never blocks the rest of the batch**. (The SDK's `insert([...])` array helper uses the NDJSON form automatically; both forms return the same response.) +A **JSON array** of objects (`[{…}, {…}]`) or an **NDJSON** body (`Content-Type: application/x-ndjson`, one JSON object per line) ingests a batch in a single request. Each record is validated, authorized, deduplicated, and published independently, so **one malformed or rejected record never blocks the rest of the batch**. (Both SDKs' array/slice insert helpers use the NDJSON form automatically; both forms return the same response.) - **JSON array** — the most convenient form from most HTTP clients. A structural JSON syntax error fails the whole request (`400`), but a wrong-typed element (a non-object) is reported per-record like any other rejection. An explicit empty array (`[]`) is a valid, record-less batch (`200`, `total: 0`). - **NDJSON** — the streaming-friendly form for very large uploads. Blank lines are skipped, and a single malformed *line* is reported and skipped (the newline reframes the next record). @@ -367,7 +367,7 @@ A `200` is returned whenever the body was read and the records were processed | 503 | `{"error":"service unavailable"}` | NATS JetStream full (backpressure) mid-batch; includes `Retry-After: 30` | :::caution[At-least-once on retry] -A batch aborted partway (a `503`/`500`, or a JSON-array syntax error, after some leading records were already published) re-publishes those leading records when the whole batch is retried. Enable deduplication if duplicate suppression matters — this is the same at-least-once property the single-object path already has (the SDK retries both on `503`). +A batch aborted partway (a `503`/`500`, or a JSON-array syntax error, after some leading records were already published) re-publishes those leading records when the whole batch is retried. Enable deduplication if duplicate suppression matters — this is the same at-least-once property the single-object path already has (the SDKs retry both on `503`). ::: **curl example (JSON array):** @@ -467,7 +467,7 @@ curl -X POST http://localhost:8080/v1/ops/query \ Executes a type-safe structured query against a table. The query AST is validated against the schema and converted to parameterized SQL. Permissions from the access control policy are enforced (column filtering, row-level security, aggregation restrictions). :::note[The column allowlist is a hard cap on every clause] -Every column the query references — in `columns`, an aggregation argument, `filters`, `group_by`, `order_by`, or `time_range` — must be permitted by the role's `allow_columns`/`deny_columns`, or the request is rejected with `403 column "x" not allowed`. A full-row read is requested with `"select_all": true` (expanded to the columns the role may read — never a raw `SELECT *`); **omitting `columns` returns nothing**, so a hidden column never leaks by being left out, grouped on, or filtered on. See [Access control → Column permissions](/access-control#column-permissions). +Every column the query references — in `columns`, an aggregation argument, `filters`, `group_by`, `order_by`, or `time_range` — must be permitted by the role's `allow_columns`/`deny_columns`, or the request is rejected with `403 column "x" not allowed`. A full-row read is requested with `"select_all": true` (for a column-restricted role, expanded to exactly the columns the role may read rather than a bare `SELECT *`; unrestricted/admin roles do get `SELECT *`); **omitting `columns` returns nothing**, so a hidden column never leaks by being left out, grouped on, or filtered on. See [Access control → Column permissions](/access-control#column-permissions). ::: **Request:** @@ -495,7 +495,7 @@ Every column the query references — in `columns`, an aggregation argument, `fi | Field | Type | Required | Description | | ----- | ---- | -------- | ----------- | | `columns` | string \| string[] | No | Columns to SELECT — an array, or a single string for one column. A literal `"*"` is the column *named* `*`, **not** a wildcard. Omit (or send `[]` / `""`) to select nothing; use `select_all` for a full-row read. Mutually exclusive with `select_all`. | -| `select_all` | bool | No | Select every column the role may read (the all-columns wildcard, expanded server-side to the allow/deny set). Mutually exclusive with a non-empty `columns`, and with `aggregations`. | +| `select_all` | bool | No | Select every column the role may read (the all-columns wildcard; a column-restricted role's projection is expanded server-side to its allow/deny set, an unrestricted/admin role gets `SELECT *`). Mutually exclusive with a non-empty `columns`, and with `aggregations`. | | `aggregations` | object[] | No | Aggregation functions (`fn`, `column`, `alias`). | | `filters` | object[] | No | WHERE conditions (`column`, `op`, `value`). Ops: eq, neq, gt, gte, lt, lte, in, like. | | `group_by` | string[] | No | GROUP BY columns. | diff --git a/docs/src/content/docs/architecture.md b/docs/src/content/docs/architecture.md index b40663ac..3b1d3ac5 100644 --- a/docs/src/content/docs/architecture.md +++ b/docs/src/content/docs/architecture.md @@ -81,7 +81,7 @@ The API layer uses [Chi](https://github.com/go-chi/chi) for routing with Request - **stream.go** — Real-time streaming via SSE. Callers select a table with the `?table=` query parameter. Each connection registers one `Subscriber` (the `stream/` package) with both the event `Hub` (under its `(topic, role)`) and the shared keepalive wheel, then drains both from a single byte-pump — so idle streams keep emitting `:` keepalive comments (surviving reverse-proxy idle timeouts) while live events arrive already projected and serialized. Per-event projection/serialization happens **once per role** in the `Hub`, not once per subscriber ([#294](https://github.com/Wave-RF/WaveHouse/issues/294)); the handler also snapshots the connection's JWT claims onto the `Subscriber`, which the `Hub` evaluates per subscriber when the role carries a row-level `filter` ([#319](https://github.com/Wave-RF/WaveHouse/issues/319)). Gap-fill replay from NATS JetStream (`DeliverByStartTime`) stays per-connection (low-volume, one-time on connect). - **schema.go** — Schema discovery API: list all schemas, get one table, trigger refresh. - **dlq.go** — DLQ stats endpoint and `EnsureDLQStream` helper for creating the `WAVEHOUSE_DLQ` NATS stream. -- **health.go** — Liveness (`/livez`), readiness (`/readyz`), and a content-free `Online` ping (`/v1/health`, the SDK's public liveness check); `/healthz` is a permanent alias of `/livez`, and `/health`/`/ready` are deprecated aliases. All three consult an optional `BootState` so they can return 503 while boot-time schema discovery is still failing in the retry loop (see `cmd/wavehouse/main.go`); once `BootState.Set(nil)` fires, `/livez` returns 200 and stays there. `/readyz` additionally pings ClickHouse each call; `/v1/health` deliberately does not. +- **health.go** — Liveness (`/livez`), readiness (`/readyz`), and a content-free `Online` ping (`/v1/health`, the SDKs' public liveness check); `/healthz` is a permanent alias of `/livez`, and `/health`/`/ready` are deprecated aliases. All three consult an optional `BootState` so they can return 503 while boot-time schema discovery is still failing in the retry loop (see `cmd/wavehouse/main.go`); once `BootState.Set(nil)` fires, `/livez` returns 200 and stays there. `/readyz` additionally pings ClickHouse each call; `/v1/health` deliberately does not. ### `stream/` — SSE keepalive & fan-out @@ -155,7 +155,7 @@ The package's design invariants — stdout always 100%, WARN+ERROR always export ### `query/` — Structured Query Engine - **ast.go** — `StructuredQuery` AST types: columns, aggregations, filters, group by, order by, limit, time range. -- **builder.go** — `Build()` converts AST to parameterized SQL. It is the single chokepoint that validates every referenced identifier against the schema **and** authorizes every column reference — projection, aggregation args, filters, group_by, order_by, time_range — against the role's column allowlist (the [#223](https://github.com/Wave-RF/WaveHouse/issues/223) hard cap). A full-row read is requested with `select_all`, which expands to the role's allowed columns rather than emitting a raw `SELECT *`; an omitted projection selects nothing, and `*` in `columns` is a literal column name. Every identifier is backtick-quoted via `internal/chsql` (`QuoteIdent`) so any ClickHouse-legal name is accepted — a name containing `?` is refused fail-closed ([#279](https://github.com/Wave-RF/WaveHouse/issues/279)). The role's row-level-security predicate and `max_rows` cap are emitted by `Build()` itself, as part of the WHERE and LIMIT assembly — policy SQL is never spliced into rendered text ([#322](https://github.com/Wave-RF/WaveHouse/issues/322)). Timestamp bucketing for cache optimization. +- **builder.go** — `Build()` converts AST to parameterized SQL. It is the single chokepoint that validates every referenced identifier against the schema **and** authorizes every column reference — projection, aggregation args, filters, group_by, order_by, time_range — against the role's column allowlist (the [#223](https://github.com/Wave-RF/WaveHouse/issues/223) hard cap). A full-row read is requested with `select_all`, which for a column-restricted role expands to the role's allowed columns rather than emitting a raw `SELECT *` (unrestricted/admin roles do get `SELECT *`); an omitted projection selects nothing, and `*` in `columns` is a literal column name. Every identifier is backtick-quoted via `internal/chsql` (`QuoteIdent`) so any ClickHouse-legal name is accepted — a name containing `?` is refused fail-closed ([#279](https://github.com/Wave-RF/WaveHouse/issues/279)). The role's row-level-security predicate and `max_rows` cap are emitted by `Build()` itself, as part of the WHERE and LIMIT assembly — policy SQL is never spliced into rendered text ([#322](https://github.com/Wave-RF/WaveHouse/issues/322)). Timestamp bucketing for cache optimization. ### `chsql/` — ClickHouse SQL Helpers @@ -283,4 +283,5 @@ Client GET /v1/stream | Embedded KV | Pebble | Optional deduplication | | Config | cleanenv | YAML + env var config loading | | Release | GoReleaser | Cross-platform binary builds | +| Client SDKs | TypeScript, Go | Typed clients with the same feature set (ingest, query, pipes, streaming, admin) | | Containers | Docker (distroless) | Minimal production images | diff --git a/docs/src/content/docs/claude-code.md b/docs/src/content/docs/claude-code.md index 59abc2d0..7e233e2b 100644 --- a/docs/src/content/docs/claude-code.md +++ b/docs/src/content/docs/claude-code.md @@ -81,7 +81,7 @@ To add a command: drop a `.md` file in `.claude/commands/`. Filename becomes the | Subagent | When to use | | -------- | ----------- | | `pre-push-reviewer` | **Mandatory before pushing to a PR branch** (enforced by `.claude/hooks/agent-bash-gate.sh`), run in parallel with the other reviewers in `scripts/pre-push-reviewers.sh` — all must reach `ship_it`. Also used for auditing someone else's PR after `wt switch pr:`. Runs the canonical `.github/prompts/pr-review.md` workflow against the local branch in fresh context. Fetches PR comments + CI status + linked-issue acceptance criteria when on a PR branch. Returns `[MUST]`/`[SHOULD]`/`[MAY]` findings + a parseable `VERDICT: ship_it\|iterate\|block` line that drives the `tmp/pre-push-reviewer-passed-` marker. | -| `docs-reviewer` | **Mandatory before pushing to a PR branch** — runs in parallel with the other pre-push reviewers (all enforced by `.claude/hooks/agent-bash-gate.sh`). Reviews docs **prose** (accuracy-vs-code, runnable examples, clarity, completeness) **and code↔docs sync** (code that changed but whose docs didn't), using `.github/prompts/docs-review.md` over the `scripts/docs-prose.sh` denylist set (Starlight site + governance docs incl. the SDK readme). Default (branch) scope emits `VERDICT: ship_it\|iterate\|block` → writes `tmp/docs-reviewer-passed-`; a path/`all` is advisory (no marker). Posts no PR comments, never edits docs. Complements misspell / markdownlint / starlight-links-validator, never duplicates them. | +| `docs-reviewer` | **Mandatory before pushing to a PR branch** — runs in parallel with the other pre-push reviewers (all enforced by `.claude/hooks/agent-bash-gate.sh`). Reviews docs **prose** (accuracy-vs-code, runnable examples, clarity, completeness) **and code↔docs sync** (code that changed but whose docs didn't), using `.github/prompts/docs-review.md` over the `scripts/docs-prose.sh` denylist set (Starlight site + governance docs incl. the SDK readmes). Default (branch) scope emits `VERDICT: ship_it\|iterate\|block` → writes `tmp/docs-reviewer-passed-`; a path/`all` is advisory (no marker). Posts no PR comments, never edits docs. Complements misspell / markdownlint / starlight-links-validator, never duplicates them. | Invoke via the `Agent` tool with `subagent_type: pre-push-reviewer`, or via `/agents`. diff --git a/docs/src/content/docs/deployment.md b/docs/src/content/docs/deployment.md index fee146d3..76c7272c 100644 --- a/docs/src/content/docs/deployment.md +++ b/docs/src/content/docs/deployment.md @@ -278,7 +278,7 @@ API servers in standalone mode expose liveness and readiness endpoints under the Configure your load balancer or orchestrator to use these endpoints. -**Exposure.** Probes share the API server's port (`:8080`) — kubelet probes the container internally, so there's no separate-port convention for them (metrics are the signal that optionally gets its own `prometheus.port`). If you forward `:8080` to the public internet the probe paths become reachable. The **recommended** posture is to keep `/livez`/`/readyz`/`/healthz` to internal callers and expose only **`/v1/health`** publicly (the SDK's content-free liveness ping, which never touches ClickHouse). `/readyz` issues a ClickHouse `Ping` on every call, so a public `/readyz` lets an unauthenticated flood become per-request backend pings, and the bare probes leak boot/readiness state — keeping them internal is a [reverse-proxy/ingress concern](/reverse-proxy#health-probes), and your orchestrator reaches them the internal way (kubelet on the container, LB on the backend) regardless. +**Exposure.** Probes share the API server's port (`:8080`) — kubelet probes the container internally, so there's no separate-port convention for them (metrics are the signal that optionally gets its own `prometheus.port`). If you forward `:8080` to the public internet the probe paths become reachable. The **recommended** posture is to keep `/livez`/`/readyz`/`/healthz` to internal callers and expose only **`/v1/health`** publicly (the SDKs' content-free liveness ping — `wh.sys.health()` / `wh.Sys.Health(ctx)` — which never touches ClickHouse). `/readyz` issues a ClickHouse `Ping` on every call, so a public `/readyz` lets an unauthenticated flood become per-request backend pings, and the bare probes leak boot/readiness state — keeping them internal is a [reverse-proxy/ingress concern](/reverse-proxy#health-probes), and your orchestrator reaches them the internal way (kubelet on the container, LB on the backend) regardless. ### Boot-time degraded mode diff --git a/docs/src/content/docs/development.md b/docs/src/content/docs/development.md index e3f765db..ea553c81 100644 --- a/docs/src/content/docs/development.md +++ b/docs/src/content/docs/development.md @@ -18,7 +18,7 @@ You need these on your `PATH` before any `make` recipe will work end-to-end: | **bash** | 4+ recommended | Recipes are pinned to `bash`; the helper scripts under `scripts/` use `set -euo pipefail` and bash arrays | macOS default is bash 3.2 (works for current recipes, but `brew install bash` is safer); Linux distros ship 4+ | | **Docker** *(or Podman)* | Engine 20.10+ with the Compose **v2** plugin (`docker compose`, no hyphen) | Compose stacks under `deployments/compose/`; the E2E and integration suites boot ClickHouse via testcontainers (no compose file) | [Docker Desktop](https://docs.docker.com/get-docker/), [colima](https://github.com/abiosoft/colima), or [Podman](https://podman.io) with `podman-compose` / the `podman compose` plugin. The testcontainers Go library also honors `DOCKER_HOST` for rootless Podman setups | | **Node.js** | 22 LTS — pinned via `.nvmrc` at the repo root | Runtime for pnpm and the Vitest suites. Pinned to match CI (`setup-node` uses 22) and to avoid Node-major surprises; older Vitest versions in this repo were known to crash on Node 26 with a V8 heap-allocation abort | [nodejs.org](https://nodejs.org/) or `nvm use` / `fnm use` / `volta` (all read `.nvmrc`) | -| **pnpm** | 11.21+ (pinned via `packageManager` in the root `package.json`) | Package manager for the TypeScript SDK, E2E test harness, and docs site (managed as a single pnpm workspace from the repo root); `make build-ts`, `make test-ts`, `make test-e2e`, `make build-docs`, `make dev-docs`, `make preview-docs` all shell out to `pnpm` | `corepack enable && corepack prepare pnpm@11.21.0 --activate` (recommended), or `npm i -g pnpm` | +| **pnpm** | 11.21+ (pinned via `packageManager` in the root `package.json`) | Package manager for the TypeScript SDK, E2E test harness, and docs site (managed as a single pnpm workspace from the repo root); `make build-ts`, `make test-sdk-ts`, `make test-e2e`, `make build-docs`, `make dev-docs`, `make preview-docs` all shell out to `pnpm` | `corepack enable && corepack prepare pnpm@11.21.0 --activate` (recommended), or `npm i -g pnpm` | | **git** + **curl** | any recent | `git` for source + version metadata in builds; `curl` is used by the Makefile to fetch the pinned `golangci-lint` binary into `.bin/` | usually preinstalled | ### Auto-installed by `make tools` @@ -41,7 +41,7 @@ node --version # v22.x (matches .nvmrc and CI) pnpm --version # 11.21+ ``` -If any of those are wrong/missing, the Makefile recipes will fail with confusing errors (e.g. `--output-sync` is unrecognized on Make 3.81; `pnpm: command not found` on `make test-ts`). +If any of those are wrong/missing, the Makefile recipes will fail with confusing errors (e.g. `--output-sync` is unrecognized on Make 3.81; `pnpm: command not found` on `make test-sdk-ts`). ### Optional but recommended @@ -186,15 +186,15 @@ They block the terminal and stream logs; simply press `Ctrl+C` to instantly tear ### Using the SDK against `make dev` -There's no bundled playground — point the published `@wavehouse/sdk` client at your local server (`baseURL: "http://localhost:8080"`), with the dev policy seeded so requests are authorized: +There's no bundled playground — point the published `@wavehouse/sdk` client (or the Go SDK, `github.com/Wave-RF/WaveHouse/clients/go`) at your local server (`baseURL: "http://localhost:8080"`), with the dev policy seeded so requests are authorized: ```bash WH_POLICY_FILE_PATH=deployments/compose/dev-policy.yaml make dev ``` -See the [SDK guide](/sdk) for the client API and examples. +See the [SDK guide](/sdk) for the client API and examples in both languages. -Frontend devs running their own dev server (Vite, Next.js, etc.) can `import { createClient } from '@wavehouse/sdk'` and point `baseURL: 'http://localhost:8080'`; CORS is permissive so cross-origin browser requests just work. +Frontend devs running their own dev server (Vite, Next.js, etc.) can `import { createClient } from '@wavehouse/sdk'` and point `baseURL: 'http://localhost:8080'`; CORS is permissive so cross-origin browser requests just work. Go services do the same with `wavehouse.NewClient(wavehouse.Config{BaseURL: "http://localhost:8080"})` — see [Go setup](/sdk/setup/go). ### Validating tokens @@ -286,35 +286,44 @@ go build -o bin/wavehouse ./cmd/wavehouse ### How It Works -All **Go** test commands use [gotestsum](https://github.com/gotestyourself/gotestsum) for pytest-style colored output with pass/fail icons, durations, and a summary. Tool versions are pinned in `go.mod` via `tool` directives — the Makefile invokes them as `go tool `, so no global installation is needed. +Every Go suite target — `test-unit`, `test-integration`, `test-sdk-go`, `test-sdk-go-e2e` — uses [gotestsum](https://github.com/gotestyourself/gotestsum) for pytest-style colored output with pass/fail icons, durations, and a summary, and every one of them honors `ARGS="..."` and `V=1`. Tool versions are pinned in `go.mod` via `tool` directives — the Makefile invokes them as `go tool `, so no global installation is needed. The nested `clients/go` module has no `tool` directives of its own, so the SDK targets resolve the binary from the root module with `go tool -n gotestsum` before running it inside `clients/go`. `test-e2e` runs the orchestrator + vitest and `test-sdk-ts` runs vitest plus the Node conformance runner, so neither uses gotestsum. -All tests run with Go's **race detector** (`-race`) enabled by default. WaveHouse is highly concurrent (NATS consumers, singleflight caching, SSE hubs) — the race detector catches data races that would panic in production. +Go tests run with the **race detector** (`-race`) enabled by default (including `test-sdk-go` — the SDK's streaming subsystem is highly concurrent; `test-sdk-go-e2e` skips it since it drives a live server). WaveHouse is highly concurrent (NATS consumers, singleflight caching, SSE hubs) — the race detector catches data races that would panic in production. ### Quick Reference ```bash -# Prefix any test target with V=1 for verbose output, e.g. `V=1 make test` +# V=1 gives verbose output on every Go suite target and test-e2e, +# e.g. `V=1 make test-unit` -# Unit tests (compact output) — alias for `test-unit` +# Go server unit tests (compact output) — alias for `test-unit` make test -# Run specific test(s) +# Run specific test(s) — ARGS passes through to go test make test ARGS="-run TestValidate" # Go integration tests (requires Docker) make test-integration -# SDK vitest unit tests + coverage + gate against suites.ts-unit -# (`make cov` auto-merges ts-unit + ts-e2e — no separate command) -make test-ts +# Both SDK suites, each including its half of the wire-format +# conformance suite +make test-sdk + +# One SDK at a time +make test-sdk-go # nested clients/go module, -race, gates suites.go-sdk +make test-sdk-ts # vitest + coverage, gates suites.ts-unit + # (`make cov` auto-merges ts-unit + ts-e2e) + +# Go SDK against a live server (nothing is started for you) +WAVEHOUSE_URL=http://localhost:8080 make test-sdk-go-e2e # E2E SDK suite against bin/wavehouse-cov make test-e2e -# All four suites sequentially + merged coverage +# All suites sequentially + merged coverage make test-all -# Full CI: parallel verify + builds (Go + SDK + docs) + test + test-ts, +# Full CI: parallel verify + builds (Go + SDK + docs) + test + test-sdk, # then test-integration + test-e2e + cov make ci @@ -322,11 +331,11 @@ make ci make cov ``` -Each test target writes `covdata` to `tmp/coverage//data/`, renders a textfmt + HTML report, and gates against the per-suite threshold in `.testcoverage.yml`. `make cov` merges whichever suites have run and gates against the total. +Each coverage-instrumented suite target (`test-unit`, `test-integration`, `test-e2e`, `test-sdk-go`, `test-sdk-ts`) writes `covdata` to `tmp/coverage//data/`, renders a textfmt + HTML report, and gates against the per-suite threshold in `.testcoverage.yml`. `make cov` merges whichever suites have run and gates against the total. `test-sdk-go` is gated but **not** merged: `clients/go` is a nested Go module, invisible to the root module's `-coverpkg=./...`, so its statements can never reach `tmp/coverage/total` — it carries its own `suites.go-sdk` floor instead, the same way the TS SDK carries `ts-*`. `test-sdk-go-e2e` runs without coverage instrumentation or a per-suite gate; giving it the same unit/e2e/total gate shape the TS SDK has is tracked in [#518](https://github.com/Wave-RF/WaveHouse/issues/518). -**Verbose output**: Use `V=1` to switch from compact `testdox` format to full verbose output. This is a standard Makefile convention (`make test -v` can't work because `-v` is a `make` flag). +**Verbose output**: Use `V=1` to switch from the compact `pkgname-and-test-fails` format to `standard-verbose` on every gotestsum target (`test-unit`, `test-integration`, `test-sdk-go`, `test-sdk-go-e2e`), and to stream live output on `test-e2e`. This is a standard Makefile convention (`make test -v` can't work because `-v` is a `make` flag). `test-sdk-ts` ignores it. -**Extra flags**: All test targets accept `ARGS="..."` for additional `go test` flags (e.g., `-run`, `-count`, `-timeout`). +**Extra flags**: every Go suite target plus `test-sdk-ts` accepts `ARGS="..."` for pass-through flags (e.g., `-run`, `-count`, `-timeout` for the Go targets; vitest flags for `test-sdk-ts`). `test-e2e` ignores it. **Note on timing**: gotestsum's `DONE ... in X.XXXs` reports pure test execution time. The total wall time includes Go compiling all packages — the first run compiles everything (~15s), subsequent runs use the build cache (~1s). @@ -335,7 +344,10 @@ Each test target writes `covdata` to `tmp/coverage//data/`, renders a tex | Category | Location | Docker? | Command | | -------- | -------- | ------- | ------- | | Unit tests | `internal/*/_test.go` | No | `make test` | -| SDK unit tests | `clients/ts/src/**/*.test.ts` | No | `make test-ts` (always includes coverage + gate) | +| SDK unit tests (TS) | `clients/ts/src/**/*.test.ts` | No | `make test-sdk-ts` (always includes coverage + gate) | +| SDK unit tests (Go) | `clients/go/*_test.go` (nested Go module) | No | `make test-sdk-go` (runs with `-race`, always includes coverage + gate) | +| Wire-format conformance | `clients/go/conformance_test.go` + `tests/conformance/conformance_ts.mjs`, both replaying `clients/go/testdata/wire_cases.json` | No | Each half rides its language's target — Go in `make test-sdk-go`, TS in `make test-sdk-ts`; `make test-sdk` runs both | +| SDK E2E (Go, live server) | `clients/go/e2e_test.go` (`//go:build e2e`) | No | `make test-sdk-go-e2e` (`WAVEHOUSE_URL`, `WAVEHOUSE_AUTH`) | | Integration tests (Go) | `tests/integration/*_test.go` | Yes | `make test-integration` | | E2E tests (SDK) | `tests/e2e/sdk/*.test.ts` | Yes | `make test-e2e` | @@ -348,7 +360,9 @@ Shared test utilities live in `internal/testutil/` (e.g., `testutil.NopLogger()` - **Unit test for `internal/foo/`** → create `internal/foo/foo_test.go` (same package). - **Integration test needing Docker** → add a subtest under `tests/integration/` (e.g. a new file with `//go:build integration`). -- **E2E test via SDK** → add a `tests/e2e/sdk/*.test.ts` file. These tests exercise the full pipeline (ingest → ClickHouse → query) through the TypeScript SDK. Run with `make test-e2e`. +- **E2E test via SDK** → add a `tests/e2e/sdk/*.test.ts` file. These tests exercise the full pipeline (ingest → ClickHouse → query) through the TypeScript SDK. Run with `make test-e2e`. (The Go SDK's own e2e suite is not yet driven by that orchestrator — [#518](https://github.com/Wave-RF/WaveHouse/issues/518).) +- **Go SDK unit test** → add to `clients/go/*_test.go` (nested module — outside `test-unit`'s scope). Run with `make test-sdk-go`. +- **Wire-format parity case** → when you add or change an endpoint, add an entry to `clients/go/testdata/wire_cases.json` plus its dispatch in both runners (`clients/go/conformance_test.go` and `tests/conformance/conformance_ts.mjs`). Required by the SDK sync rule in `AGENTS.md` / `CONTRIBUTING.md`. - **Test helpers** → add to `internal/testutil/` (Go) or `tests/e2e/sdk/helpers.ts` (E2E). ### E2E Tests via SDK @@ -460,13 +474,16 @@ WaveHouse/ │ ├── query/ # Structured query AST + SQL builder │ ├── stream/ # SSE fan-out: Hub, Subscriber queue, Bucket, keepalive wheel │ └── testutil/ # Shared test helpers and mocks +├── clients/ # Official SDKs +│ ├── ts/ # TypeScript SDK (@wavehouse/sdk, pnpm workspace) +│ └── go/ # Go SDK — a NESTED Go module (own go.mod, invisible +│ # to root `go list`; hence the test-sdk-go* targets) ├── tests/ # Integration & E2E tests │ ├── integration/ # Go integration tests (//go:build integration) +│ ├── conformance/ # TS half of the cross-SDK wire-format conformance suite │ └── e2e/ # E2E suite (orchestrator + ClickHouse testcontainer) │ ├── fixtures/ # ClickHouse DDL + config/policy fixtures │ └── sdk/ # E2E specs driven through the TypeScript SDK (Vitest) -├── clients/ # Client SDKs -│ └── ts/ # TypeScript SDK (@wavehouse/sdk, pnpm workspace) ├── deployments/ │ ├── compose/ # Docker Compose files (standalone.yaml, dependencies.yaml) │ ├── Dockerfile # Runtime image @@ -508,11 +525,11 @@ Run `make help` to see all targets. Key ones: | `make obs-grafana` | Grafana alternative to aspire, more advanced and complicated | | `make obs-front` | Custom graphs like grafana, but is simpler and easier to configure like aspire | | **Static checks** | | -| `make fmt` | Check formatting across Go (`gofumpt`) + TS (Biome). Run `make fix` to apply. | -| `make tidy` | Verify `go.mod`/`go.sum` are tidy (run `make fix` to apply) | -| `make lint` | Run linters across Go (`golangci-lint`) + TS (Biome) + Markdown/MDX (markdownlint) + prose (misspell) | +| `make fmt` | Check formatting across Go (`gofumpt`, both the root module and the nested `clients/go`) + TS (Biome). Run `make fix` to apply. | +| `make tidy` | Verify `go.mod`/`go.sum` are tidy in both modules (run `make fix` to apply) | +| `make lint` | Run linters across Go (`golangci-lint`, root + `clients/go`) + TS (Biome) + Markdown/MDX (markdownlint) + prose (misspell) | | `make vulncheck` | Run `govulncheck` (V=1 for full call stacks) | -| `make verify` | Repo-wide static checks: Go (tidy + fmt + vulncheck + lint) + TS (Biome + `tsc` typecheck) + Markdown/MDX (markdownlint + rule fixtures) + prose (misspell) + shell (shellcheck) + workflows (actionlint) + path-classifier fixtures + release-channel fixtures + docs type-check (`astro check` — not a full build, so link validation stays CI's job) (parallel-safe: `make -j verify`) | +| `make verify` | Repo-wide static checks: Go (tidy + fmt + lint across both modules; `vulncheck` is still root-only — extending it to the nested module is tracked in [#437](https://github.com/Wave-RF/WaveHouse/issues/437)) + TS (Biome + `tsc` typecheck) + Markdown/MDX (markdownlint + rule fixtures) + prose (misspell) + shell (shellcheck) + workflows (actionlint) + path-classifier fixtures + release-channel fixtures + docs type-check (`astro check` — not a full build, so link validation stays CI's job) (parallel-safe: `make -j verify`) | | `make fix` | Auto-fixes across Go (`tidy` + `gofumpt` + `goimports` + `lint --fix`), TS (Biome `--write`), Markdown (markdownlint `--fix`), MDX (`fix-mdx-fences` only — the generic fixers never run over `.mdx`), and docs-prose spelling (misspell, both) | | **Build** | | | `make build` | Compile `wavehouse` → `bin/wavehouse` (debug symbols kept) | @@ -520,18 +537,21 @@ Run `make help` to see all targets. Key ones: | `make build-cover` | Coverage-instrumented build → `bin/wavehouse-cov` (used by E2E) | | `make build-ts` | Build TypeScript SDK → `clients/ts/dist/` | | **Test** | | -| `make test` | Alias for `test-unit` | +| `make test` | Alias for `test-unit` (Go server unit tests) | | `make test-unit` | Go unit tests + render coverage + gate suite threshold | +| `make test-sdk` | Both SDK suites — `test-sdk-go` + `test-sdk-ts` | +| `make test-sdk-go` | Go SDK (`clients/go`, nested module) unit + wire-conformance tests with `-race` + render coverage + gate `suites.go-sdk` (own gate; never merged into the Go total) | +| `make test-sdk-ts` | TS SDK vitest unit + wire-conformance tests + v8 coverage + gate against `suites.ts-unit` (matches Go's "always coverage" pattern) | +| `make test-sdk-go-e2e` | Go SDK E2E against a live server (`WAVEHOUSE_URL`, `WAVEHOUSE_AUTH`) | | `make test-integration` | Go integration tests (requires Docker) + coverage gate | -| `make test-ts` | SDK vitest unit tests + v8 coverage + gate against `suites.ts-unit` (matches Go's "always coverage" pattern) | | `make cov` | Merge Go + TS coverage and gate against thresholds. Auto-runs after `make test-all` and `make ci`; standalone `make cov` is "show me the merged numbers without re-running." Each side skips silently if its data is missing, but `make cov` fails if *both* are empty (you ran it before any test target). | | `make test-e2e` | E2E SDK suite against `bin/wavehouse-cov` + coverage gate | -| `make test-all` | All four suites sequentially + merged coverage gate | -| `make ci` | Full pipeline: parallel `verify` + builds + unit/SDK tests, then integration + E2E + cov | +| `make test-all` | All suites sequentially + merged coverage gate | +| `make ci` | Full pipeline: parallel `verify` + builds + `test` / `test-sdk`, then integration + E2E + cov | | **Release** (see [Cutting a release](#cutting-a-release)) | | | `make release-server VERSION=X.Y.Z` | Tag a server release — binaries + container image | | `make release-sdk-ts VERSION=X.Y.Z` | Tag a TypeScript SDK release — npm | -| `make release-sdk-go VERSION=X.Y.Z` | Tag a Go SDK release — `go get` (pending [#434](https://github.com/Wave-RF/WaveHouse/pull/434)) | +| `make release-sdk-go VERSION=X.Y.Z` | Tag a Go SDK release — `go get` | | **Analysis** (informational, not in CI) | | | `make size` | Binary size analysis → `tmp/analysis/` (text + SVG + interactive HTML) | | `make audit-cgo` | Audit dependency tree for C files (builds use `CGO_ENABLED=0`) | @@ -544,7 +564,7 @@ Run `make help` to see all targets. Key ones: | `make clean-tools` | Installed tools and pnpm deps (`.bin/`, `node_modules/`) | | `make clean-all` | Full reset: above + `data/` + Docker volumes | -All test targets accept `ARGS="..."` for pass-through `go test` flags. Build targets accept `TAGS="..."` for Go build tags. `V=1` switches to verbose `gotestsum` output. +Every Go suite target (`test-unit`, `test-integration`, `test-sdk-go`, `test-sdk-go-e2e`) plus `test-sdk-ts` accepts `ARGS="..."` for pass-through flags; those Go targets and `test-e2e` accept `V=1` for verbose output, which `test-sdk-ts` ignores. Build targets accept `TAGS="..."` for Go build tags. ## Dependency Management @@ -588,10 +608,9 @@ Cutting a release is **one tag** — no version bump in code, no release branch: ```bash make release-server VERSION=0.1.0 # tag v0.1.0 → binaries + container image make release-sdk-ts VERSION=0.1.0 # tag clients/ts/v0.1.0 → @wavehouse/sdk on npm +make release-sdk-go VERSION=0.1.0 # tag clients/go/v0.1.0 → the Go module proxy ``` -`make release-sdk-go` exists too, wired ahead of the Go SDK landing ([#434](https://github.com/Wave-RF/WaveHouse/pull/434)); it refuses to run until `clients/go/` is in the repo. - The one thing to do *before* tagging is promote the changelog: `AGENTS.md` requires every PR to add its entry under `## Unreleased`, so open a PR renaming that heading to `## [X.Y.Z] - YYYY-MM-DD` and adding the matching link reference at the foot of the file. Nothing in the release pipeline reads `CHANGELOG.md` — this is for the file's own readers. Each runs [`scripts/release.sh`](https://github.com/Wave-RF/WaveHouse/blob/main/scripts/release.sh), which preflights (on `main`, clean tree, in sync with `origin/main`, the tag free both locally and on the remote, the required `CI` check green on *this exact commit*), prints exactly what will be published, and asks before pushing. `DRY_RUN=1 make release-…` stops after the plan. Tag creation is admin-only via the `release tag protection` ruleset. @@ -625,9 +644,10 @@ Tag globs are anchored at the start of the ref name, so `v*` never matches a `cl ### What a release publishes - **Server —** a **GitHub Release** with the cross-compiled archives (linux/darwin/windows/freebsd × amd64/arm64; `.zip` on Windows, `.tar.gz` elsewhere) and `checksums.txt`. A tag carrying a prerelease suffix (`v0.1.0-alpha.1`) is marked as a GitHub pre-release, so it never takes the "Latest release" badge from a shipped stable version. -- **Both —** **release notes generated by GitHub** from the PRs merged since the previous tag *in the same family* — one line per PR, since `main` is squash-merged, grouped into the categories defined in [`.github/release.yml`](https://github.com/Wave-RF/WaveHouse/blob/main/.github/release.yml). Grouping is by **PR label**: `github_actions` / `documentation` are applied automatically by `actions/labeler`, but `breaking-change`, `security`, `bug`, and `enhancement` are applied by hand — an unlabelled PR lands in "Other changes". Dependabot is split out by **author** rather than by label, because the labels `actions/labeler` applies by path — `github_actions`, `documentation` — mark our own PRs too; our CI work gets its own "CI & build" section — ordered above Documentation, since a CI PR here nearly always updates docs too — and Dependencies is pure Dependabot residue. **Any category keyed on a label a Dependabot PR can carry needs that author exclude** — labeler's path labels *and* the ecosystem labels Dependabot applies itself (`dependencies`, `javascript`, `go`, `github_actions`; `javascript` is in neither `labeler.yml` nor our categories) — or that category intercepts bumps before they reach the `📦 Dependencies` catch-all. `CHANGELOG.md` is *not* the source of the release body; it is the longer-form record of why each change was made. +- **Server + TypeScript SDK —** **release notes generated by GitHub** from the PRs merged since the previous tag *in the same family* — one line per PR, since `main` is squash-merged, grouped into the categories defined in [`.github/release.yml`](https://github.com/Wave-RF/WaveHouse/blob/main/.github/release.yml). Grouping is by **PR label**: `github_actions` / `documentation` are applied automatically by `actions/labeler`, but `breaking-change`, `security`, `bug`, and `enhancement` are applied by hand — an unlabelled PR lands in "Other changes". Dependabot is split out by **author** rather than by label, because the labels `actions/labeler` applies by path — `github_actions`, `documentation` — mark our own PRs too; our CI work gets its own "CI & build" section — ordered above Documentation, since a CI PR here nearly always updates docs too — and Dependencies is pure Dependabot residue. **Any category keyed on a label a Dependabot PR can carry needs that author exclude** — labeler's path labels *and* the ecosystem labels Dependabot applies itself (`dependencies`, `javascript`, `go`, `github_actions`; `javascript` is in neither `labeler.yml` nor our categories) — or that category intercepts bumps before they reach the `📦 Dependencies` catch-all. `CHANGELOG.md` is *not* the source of the release body; it is the longer-form record of why each change was made. - **Server —** a **GHCR image** at `ghcr.io/wave-rf/wavehouse`, with two tags: the immutable `:vX.Y.Z`, and one moving *channel* pointer. A stable release moves `:latest`; a prerelease moves `:alpha` / `:beta` / `:rc` / `:next` instead, matching the npm dist-tag it would get. The channel comes from the **first** prerelease identifier, matched **exactly**: `v0.2.0-rc.1` → `:rc`, while `-alpha1`, `-preview.1`, or any other form → `:next`. `scripts/ci/release-channel.sh` is the single rule every publisher uses, so `ghcr.io/wave-rf/wavehouse:rc` and `@wavehouse/sdk@rc` can't drift apart. **A prerelease-only project therefore has no `:latest` tag** — that is deliberate; `:latest` starts existing when the first stable release ships. - **TypeScript SDK —** an **npm publish** of `@wavehouse/sdk` under `latest` (stable) or `alpha`/`beta`/`rc`/`next` (prerelease), plus its own GitHub Release. +- **Go SDK —** nothing to upload. A `clients/go/vX.Y.Z` tag *is* the release: the [Go module proxy](https://proxy.golang.org) serves it on the first `go get github.com/Wave-RF/WaveHouse/clients/go@vX.Y.Z`, and `sum.golang.org` records the module hash. No workflow fires — the release tag globs (`v*`, `clients/ts/v*`) are anchored and never match it — so there is no GitHub Release or provenance attestation for the Go SDK today. - **Server —** **build-provenance attestations** (Sigstore, free for public repos) over every archive and over the image's multi-arch manifest digest; the image attestation is stored alongside the image in GHCR. The release job verifies its own attestations before finishing, so a release that publishes unverifiable provenance goes red. - **TypeScript SDK —** an **npm provenance attestation** via `npm publish --provenance`, surfaced as the provenance badge on the package page and checkable with `npm audit signatures`. The npm job does not re-verify it the way the server job does. @@ -688,7 +708,7 @@ If the title doesn't match, a sticky comment posts on the PR explaining the form The `main branch protection` ruleset requires one status check to pass before any PR can merge: -- `CI` — the aggregator job of `.github/workflows/ci.yml`. The workflow is a job DAG over the same Makefile targets local `make ci` runs: `lint` (`make verify`), `unit` (`make test-unit test-ts`), `integration` (`make test-integration`), `e2e` (`make -j test-e2e` — builds its own SDK dist + cover binary on a warm cache, runs the suite exactly like a local run), `coverage` (`make cov` over every suite's uploaded coverage fragment + threshold gates, like local `make ci`'s final step), `docs-build` (`make build-docs` when docs-affecting files changed, uploading the docs dist artifact), `PR title` (Conventional Commits), and the docs preview/deploy jobs. The aggregator fails if any job failed or was canceled and treats skipped jobs as passing — docs-only PRs skip the Go test suites by design, and fork PRs run everything except the (secret-bearing) docs deploys. Every run's Summary page gets a per-job wall-clock table from the non-gating `Timing summary` job. The full architecture — DAG diagram, design invariants, cache policy, how to add a job — lives in [`.github/workflows/README.md`](https://github.com/Wave-RF/WaveHouse/blob/main/.github/workflows/README.md). +- `CI` — the aggregator job of `.github/workflows/ci.yml`. The workflow is a job DAG over the same Makefile targets local `make ci` runs: `lint` (`make verify`), `unit` (`make test-unit test-sdk`), `integration` (`make test-integration`), `e2e` (`make -j test-e2e` — builds its own SDK dist + cover binary on a warm cache, runs the suite exactly like a local run), `coverage` (`make cov` over every suite's uploaded coverage fragment + threshold gates, like local `make ci`'s final step), `docs-build` (`make build-docs` when docs-affecting files changed, uploading the docs dist artifact), `PR title` (Conventional Commits), and the docs preview/deploy jobs. The aggregator fails if any job failed or was canceled and treats skipped jobs as passing — docs-only PRs skip the Go test suites by design, and fork PRs run everything except the (secret-bearing) docs deploys. Every run's Summary page gets a per-job wall-clock table from the non-gating `Timing summary` job. The full architecture — DAG diagram, design invariants, cache policy, how to add a job — lives in [`.github/workflows/README.md`](https://github.com/Wave-RF/WaveHouse/blob/main/.github/workflows/README.md). The `PR housekeeping` workflow still runs on every PR (labels + the title explainer comment) but is no longer a required check. diff --git a/docs/src/content/docs/getting-started.md b/docs/src/content/docs/getting-started.md index 8b368d8f..3339b598 100644 --- a/docs/src/content/docs/getting-started.md +++ b/docs/src/content/docs/getting-started.md @@ -76,7 +76,7 @@ curl -s -X POST "http://localhost:8080/v1/query?table=clicks" \ `POST /v1/query?table={table}` and `GET/POST /v1/pipes/{name}` are cached in-process (L1 Ristretto) with singleflight coalescing — duplicate concurrent queries hit ClickHouse once. For raw SQL there's `POST /v1/ops/query` (an admin escape hatch that never caches, emitting `Cache-Control: no-store`), but it's **admin-only** — the trial `public` role can't reach it. To use it, swap the public default for real auth: configure a JWT secret and present a token whose role is the policy [`admin_role`](/access-control#admin_role--the-privileged-role). :::tip[Prefer a type-safe client?] -The [TypeScript SDK](/sdk) wraps this endpoint in a chainable query builder with autocomplete on your table names and row types — plus live queries and streaming. The raw shapes are in the [structured query reference](/api#post-v1querytabletable--structured-query). +The [client SDKs](/sdk/queries) wrap this endpoint in a chainable query builder — autocomplete on your table names and row types in TypeScript, generics for typed rows in Go — plus live queries and streaming. The raw shapes are in the [structured query reference](/api#post-v1querytabletable--structured-query). ::: ## 5. Subscribe to real-time updates @@ -105,7 +105,7 @@ The handful of things that most often trip up a first session — each is expect - **[Architecture](/architecture)** — how ingest, query, cache, and streaming fit together. - **[API Reference](/api)** — every endpoint, request/response shape, and error code. -- **[TypeScript SDK](/sdk)** — client with query builder, live queries, and codegen; one runtime dependency. +- **[Client SDKs](/sdk)** — official TypeScript and Go clients: query builder, live queries, streaming, and typed rows. - **[Configuration](/configuration)** — full YAML + environment variable reference. - **[Deployment](/deployment)** — Docker images, releases, health checks. - **[Development](/development)** — building from source, running tests, hot-reload workflow. diff --git a/docs/src/content/docs/index.mdx b/docs/src/content/docs/index.mdx index 0fe6c504..1275e4fb 100644 --- a/docs/src/content/docs/index.mdx +++ b/docs/src/content/docs/index.mdx @@ -102,8 +102,8 @@ If you're building user-facing analytics, **WaveHouse is like Supabase for Click Per-table, per-role column and row-level policies with JWT claim templating. Stored in NATS KV with file-based bootstrap and cluster sync. - - `@wavehouse/sdk` — a type-safe query builder, live queries, real-time streaming, and codegen from your schemas, with one runtime dependency of ~1.4 KB gzipped. + + `@wavehouse/sdk` and `github.com/Wave-RF/WaveHouse/clients/go` — type-safe query builders, live queries, real-time streaming, and typed rows, speaking one shared wire format. @@ -111,7 +111,7 @@ If you're building user-facing analytics, **WaveHouse is like Supabase for Click ## Query it like a database. Subscribe to it like a socket -The [TypeScript SDK](/sdk) wraps the whole surface — typed inserts, a chainable query builder, and live queries that backfill history before streaming: +The [client SDKs](/sdk) wrap the whole surface — typed inserts, a chainable query builder, and live queries that backfill history before streaming. Examples below are TypeScript; the Go client mirrors the same feature set, and every SDK page carries both languages side by side. @@ -212,8 +212,8 @@ Self-hosting WaveHouse is deliberately boring — one binary, one dependency. Bu href="/api" /> diff --git a/docs/src/content/docs/pipes.mdx b/docs/src/content/docs/pipes.mdx index 78c753cf..5b6336cf 100644 --- a/docs/src/content/docs/pipes.mdx +++ b/docs/src/content/docs/pipes.mdx @@ -148,7 +148,7 @@ curl -X PUT http://localhost:8080/v1/ops/pipes/top_pages \ }' ``` -A `PUT` is a full replace of that named pipe; `name` is taken from the URL. Definitions are stored in NATS KV and synced across nodes, so a create/update/delete applies cluster-wide without a restart. The [TypeScript SDK](/sdk) exposes the same operations as `client.pipes.list()`, `client.pipes.get(name)`, `client.pipes.set(name, def)`, and `client.pipes.delete(name)`. +A `PUT` is a full replace of that named pipe; `name` is taken from the URL. Definitions are stored in NATS KV and synced across nodes, so a create/update/delete applies cluster-wide without a restart. Both [SDKs](/sdk/pipes#managing-pipe-definitions) expose the same operations — TypeScript as `client.pipes.list()`, `client.pipes.get(name)`, `client.pipes.set(name, def)`, and `client.pipes.delete(name)`, Go as `wh.Pipes.List(ctx)`, `wh.Pipes.Get(ctx, name)`, `wh.Pipes.Set(ctx, name, def)`, and `wh.Pipes.Delete(ctx, name)`. ## Executing a pipe diff --git a/docs/src/content/docs/reverse-proxy.mdx b/docs/src/content/docs/reverse-proxy.mdx index 63e2ea95..539f9e1b 100644 --- a/docs/src/content/docs/reverse-proxy.mdx +++ b/docs/src/content/docs/reverse-proxy.mdx @@ -82,7 +82,7 @@ handle_path /api/wavehouse/* { The ingress controller forwards the full path unless you ask it to rewrite. Pair a capture-group path (`/api/wavehouse(/|$)(.*)` with `pathType: ImplementationSpecific`) with the `nginx.ingress.kubernetes.io/rewrite-target: /$2` annotation, or the prefix arrives at WaveHouse unstripped. ::: -Point the SDK at the prefixed URL and it does the rest — `createClient({ baseURL: 'https://app.example.com/api/wavehouse' })` sends both REST calls and SSE streams under the prefix ([SDK → Serving under a path prefix](/sdk#serving-under-a-path-prefix)). +Point either SDK at the prefixed URL and it does the rest — `createClient({ baseURL: 'https://app.example.com/api/wavehouse' })` in TypeScript, `wavehouse.NewClient(wavehouse.Config{BaseURL: "https://app.example.com/api/wavehouse"})` in Go — both sending REST calls and SSE streams under the prefix ([TypeScript → Serving under a path prefix](/sdk/setup/typescript#serving-under-a-path-prefix), [Go → Creating a client](/sdk/setup/go#creating-a-client)). ## Request-body size limits @@ -209,10 +209,10 @@ WaveHouse serves Kubernetes-convention probes on `:8080` (full behavior in [Depl - **`/livez`** — liveness; sticky-200 after first successful boot. Does not touch ClickHouse. - **`/readyz`** — readiness; issues a ClickHouse `Ping` on **every** call. Point your load balancer's (internal) health check here so it routes around an instance whose ClickHouse is unreachable. - **`/healthz`** — permanent alias of `/livez`. -- **`/v1/health`** — the SDK's content-free liveness ping; mirrors `/livez` (200 once booted) and never touches ClickHouse. +- **`/v1/health`** — the SDKs' content-free liveness ping; mirrors `/livez` (200 once booted) and never touches ClickHouse. :::caution[Recommended: keep the bare probe paths off the public vhost] -Expose only **`/v1/health`** (and your API) to the internet — route `/livez`, `/readyz`, and `/healthz` on a private/internal listener, not the public proxy. Your orchestrator still reaches them the internal way regardless: kubelet probes the container directly on `:8080`, and a load balancer health-checks the backend — neither goes through the public proxy. Two reasons to keep them internal: `/readyz` pings ClickHouse on every call, so a public `/readyz` lets an unauthenticated flood turn into a per-request backend ping; and the probes leak boot/readiness state. `/v1/health` is the safe public liveness endpoint because it answers the same "is this server up" question without touching ClickHouse — it's what the SDK's `wh.sys.health()` calls. +Expose only **`/v1/health`** (and your API) to the internet — route `/livez`, `/readyz`, and `/healthz` on a private/internal listener, not the public proxy. Your orchestrator still reaches them the internal way regardless: kubelet probes the container directly on `:8080`, and a load balancer health-checks the backend — neither goes through the public proxy. Two reasons to keep them internal: `/readyz` pings ClickHouse on every call, so a public `/readyz` lets an unauthenticated flood turn into a per-request backend ping; and the probes leak boot/readiness state. `/v1/health` is the safe public liveness endpoint because it answers the same "is this server up" question without touching ClickHouse — it's what the SDKs' liveness ping calls (`wh.sys.health()` / `wh.Sys.Health(ctx)`). ::: ## Timeouts and slow links diff --git a/docs/src/content/docs/sdk/admin.md b/docs/src/content/docs/sdk/admin.md deleted file mode 100644 index c1523fb9..00000000 --- a/docs/src/content/docs/sdk/admin.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -title: "SDK Admin & System" -description: "Schema introspection, access-control policy, DLQ stats, and health checks in @wavehouse/sdk." ---- - -Operational surfaces of `@wavehouse/sdk`. With one exception, everything on this page sits behind the server's admin gate: the caller must resolve to the admin role (`policy.admin_role`) or present the non-JWT [operator key](/api#authentication) — the SDK has no first-class operator-key option, but [`options.headers`](/sdk#custom-headers) can carry the `X-Operator-Key` header. The exception is `wh.sys.health()`, which calls the public, content-free `/v1/health` route and needs no credentials. See [Access Control](/access-control) for how roles resolve. Examples import from `@wavehouse/sdk` or `https://esm.sh/@wavehouse/sdk` (see [Imports & Runtimes](/sdk#imports--runtimes)). - -## Schema — `wh.schema` - -Introspect ClickHouse table schemas. - -```ts -// List all table schemas -const { data: schemas } = await wh.schema.list(); -// schemas: { clicks: { name: 'clicks', columns: [...] }, users: { ... } } - -// Force refresh from ClickHouse -await wh.schema.refresh(); -``` - -Individual table schema is also available via `wh.from('clicks').schema()`. - -> `wh.schema.list()`, `wh.schema.refresh()`, and `wh.from(t).schema()` hit `/v1/ops/schema*`, which are **admin-only** endpoints: the caller must pass the admin gate — resolve to the policy admin role (`admin_role`, `"admin"` by default) or present the non-JWT [operator key](/api#authentication). Unless the deployment deliberately sets `default_role` to the admin role (the loudly-warned dev-only setting), construct the client with an admin-role token — or send the operator key via [`options.headers`](/sdk#custom-headers) — or these calls return `403`. - ---- - -## Policy — `wh.policy` - -Manage Hasura-style access control policies. Requires the admin gate — the admin role (`policy.admin_role`) or the [operator key](/api#authentication). - -```ts -// Get current policy -const { data: policy } = await wh.policy.get(); - -// Update policy -await wh.policy.set({ - default_role: 'viewer', - admin_role: 'admin', - tables: { - clicks: { - select: { - viewer: { - allow_columns: ['page', 'button', 'received_timestamp'], - filter: { tenant_id: { _eq: '{{ jwt.app_metadata.tenant_id }}' } }, - }, - admin: { allow_columns: ['*'] }, - }, - }, - }, -}); - -// Validate without applying (dry run) -const { data } = await wh.policy.validate(policyDraft); -// data: { valid: true } or error with validation details -``` - ---- - -## DLQ — `wh.dlq` - -Dead Letter Queue operations. Requires the admin gate — the admin role (`policy.admin_role`) or the [operator key](/api#authentication). - -```ts -// Get DLQ statistics -const { data } = await wh.dlq.list(); -// data: { tables: { "clicks": 3, "users": 0 }, total: 3 } - -// Stats for a specific table -const { data } = await wh.dlq.table('clicks'); -``` - -`wh.dlq.stream()` exists in the API but is **not yet functional**: there is no server-side DLQ stream today (the SSE bridge only carries `ingest.>` subjects), so it connects and receives no events rather than failing. Live DLQ streaming is tracked in [#197](https://github.com/Wave-RF/WaveHouse/issues/197). - ---- - -## System — `wh.sys` - -Content-free server-online check. - -```ts -// health() hits the public, content-free /v1/health route — 200/503, no body. -// Use it to check a server is reachable before sending data. -const result = await wh.sys.health(); -if (result.ok) { - // server is up and past boot -} -// on failure, result.error carries the reason (network vs. server error) -``` - -> Readiness (`/readyz`) is intentionally **not** exposed through the SDK — it runs a ClickHouse query per call and is a load-balancer / reverse-proxy concern, not the client's. Probe `/readyz` directly from your orchestrator if you need it. diff --git a/docs/src/content/docs/sdk/admin.mdx b/docs/src/content/docs/sdk/admin.mdx new file mode 100644 index 00000000..1199f6a7 --- /dev/null +++ b/docs/src/content/docs/sdk/admin.mdx @@ -0,0 +1,191 @@ +--- +title: "SDK Admin & System" +description: "Schema introspection, access-control policy, DLQ stats, and health checks in the WaveHouse SDKs." +--- + +import { Tabs, TabItem } from "@astrojs/starlight/components"; + +Operational surfaces of the SDKs. With one exception, everything on this page sits behind the server's admin gate on `/v1/ops/*`: the caller must resolve to the admin role (`policy.admin_role`, `"admin"` by default) or present the server's non-JWT [operator key](/api#authentication) as an `X-Operator-Key` header. Without one, these calls return `403`, unless the deployment deliberately sets `default_role` to the admin role, a loudly-warned dev-only setting. The exception is the health check, which calls the public, content-free `/v1/health` route and needs no credentials. See [Access Control](/access-control) for how roles resolve. + +Neither SDK has a first-class operator-key option; each carries it as a custom header: [`options.headers`](/sdk/setup/typescript#custom-headers) in TypeScript, [`ClientOptions.Headers`](/sdk/setup/go#clientoptions) in Go. + +## Schema + +Introspect ClickHouse table schemas. A single table's schema is also available from its table ref (see [Queries → Table schema](/sdk/queries#table-schema)). + + + + +```ts +// List all table schemas +const { data: schemas } = await wh.schema.list(); +// schemas: { clicks: { name: 'clicks', columns: [...] }, users: { ... } } + +// Force refresh from ClickHouse +await wh.schema.refresh(); + +// One table +const { data } = await wh.from('clicks').schema(); +``` + + + + +```go +// List all table schemas. +schemas, err := wh.Schema.List(ctx) +// schemas is wavehouse.Schemas: map[string]TableSchema, keyed by table name + +// Force refresh from ClickHouse. +err = wh.Schema.Refresh(ctx) + +// One table. +schema, err := wh.From("clicks").Schema(ctx) +``` + + + + +--- + +## Policy + +Manage Hasura-style access control policies. + + + + +```ts +// Get current policy +const { data: policy } = await wh.policy.get(); + +// Update policy +const policyDraft = { + default_role: 'viewer', + admin_role: 'admin', + tables: { + clicks: { + select: { + viewer: { + allow_columns: ['page', 'button', 'received_timestamp'], + filter: { tenant_id: { _eq: '{{ jwt.app_metadata.tenant_id }}' } }, + }, + admin: { allow_columns: ['*'] }, + }, + }, + }, +}; +await wh.policy.set(policyDraft); + +// Validate without applying (dry run) +const { data } = await wh.policy.validate(policyDraft); +// data: { valid: true } or error with validation details +``` + + + + +```go +// Get current policy. +policy, err := wh.Policy.Get(ctx) + +// Update policy. +tenantFilter := "{{ jwt.app_metadata.tenant_id }}" +policyDraft := &wavehouse.Policy{ + DefaultRole: "viewer", + Tables: map[string]wavehouse.TablePolicy{ + "clicks": { + Select: map[string]wavehouse.RolePermissions{ + "viewer": { + AllowColumns: []string{"page", "button", "received_timestamp"}, + Filter: map[string]wavehouse.PolicyFilter{ + "tenant_id": {Eq: &tenantFilter}, + }, + }, + "admin": {AllowColumns: []string{"*"}}, + }, + }, + }, +} +err = wh.Policy.Set(ctx, policyDraft) + +// Validate without applying (dry run). +result, err := wh.Policy.Validate(ctx, policyDraft) +// result.Valid == true, or err wraps the validation failure details +``` + +`PolicyFilter` fields (`Eq`, `Neq`, `Gt`, `Lt`, `In`) are `*string`, so an empty string is distinguishable from an absent operator, hence the `tenantFilter` variable above, or `wavehouse.Ptr("...")`. + + + + +--- + +## Dead Letter Queue + +Dead Letter Queue statistics. The DLQ stream method exists in both API trees but is **not yet functional**: there is no server-side DLQ stream today (the SSE bridge only carries `ingest.>` subjects), so it connects and receives no events rather than failing. Live DLQ streaming is tracked in [#197](https://github.com/Wave-RF/WaveHouse/issues/197). + + + + +```ts +// Get DLQ statistics +const { data } = await wh.dlq.list(); +// data: { tables: { "clicks": 3, "users": 0 }, total: 3 } + +// Stats for a specific table +const { data } = await wh.dlq.table('clicks'); +``` + + + + +```go +// Totals across tables. +stats, err := wh.DLQ.List(ctx) +// stats.Tables: map[string]int{"clicks": 3, "users": 0} +// stats.Total: 3 + +// Stats for a specific table. +stats, err = wh.DLQ.Table(ctx, "clicks") +``` + + + + +--- + +## System + +Content-free server-online check: the one surface on this page that needs no credentials. + + + + +```ts +// health() hits the public, content-free /v1/health route: 200/503, no body. +// Use it to check a server is reachable before sending data. +const result = await wh.sys.health(); +if (result.ok) { + // server is up and past boot +} +// on failure, result.error carries the reason (network vs. server error) +``` + + + + +```go +// Health hits the public, content-free /v1/health route: 200 → nil error, +// any other status (including 503) → a non-nil *wavehouse.Error. +// Use it to check a server is reachable before sending data. +if err := wh.Sys.Health(ctx); err != nil { + // server is unreachable or not yet past boot + log.Println(err) +} +``` + + + + +> Readiness (`/readyz`) is intentionally **not** exposed through either SDK: it runs a ClickHouse query per call and is a load-balancer / reverse-proxy concern, not the client's. Probe `/readyz` directly from your orchestrator if you need it. diff --git a/docs/src/content/docs/sdk/index.mdx b/docs/src/content/docs/sdk/index.mdx index f5bc73bc..1baff9db 100644 --- a/docs/src/content/docs/sdk/index.mdx +++ b/docs/src/content/docs/sdk/index.mdx @@ -1,538 +1,100 @@ --- -title: "TypeScript SDK" -description: "Client SDK — query builder, real-time streaming, codegen." +title: "Client SDKs" +description: "Official WaveHouse clients for TypeScript and Go: query builder, real-time streaming, typed rows." --- import { Tabs, TabItem, LinkCard, CardGrid } from "@astrojs/starlight/components"; -`@wavehouse/sdk` — TypeScript client for WaveHouse. One runtime dependency: `eventsource-parser` (~1.4 KB gzipped, itself dependency-free), which frames the SSE stream. +WaveHouse ships two officially supported clients with full API-tree parity: **TypeScript** (`@wavehouse/sdk`) and **Go** (`github.com/Wave-RF/WaveHouse/clients/go`). Both give you a typed query builder, real-time SSE streaming over header-authenticated HTTP, live queries that backfill history before going live, and typed rows: TypeScript generates them from your ClickHouse schema, Go takes a struct you write. -## Installation +They speak the same wire format, and a shared fixture replayed by both test suites keeps it that way, so the topic pages below cover both languages side by side. Pick a language once and the tabs follow you across every page. - - +## Install -```bash -pnpm add @wavehouse/sdk -``` - - - + + ```bash npm install @wavehouse/sdk ``` - - - -```bash -yarn add @wavehouse/sdk -``` +Other package managers, the CDN build, and runtime support: [TypeScript setup](/sdk/setup/typescript#installation). - + ```bash -bun add @wavehouse/sdk +go get github.com/Wave-RF/WaveHouse/clients/go ``` - - - -```bash -deno add npm:@wavehouse/sdk -``` - - - - No bundler or package manager required — the ES module loads natively in modern browsers: - -```html - -``` - -Pin a version for production (`https://esm.sh/@wavehouse/sdk@0.1.0`); jsDelivr (`.../+esm`) and unpkg (`?module`) serve the same module. For pages that can't use ES modules, the bundled IIFE build at `https://cdn.jsdelivr.net/npm/@wavehouse/sdk` exposes a `WaveHouse` global (`WaveHouse.createClient({ … })`) for a classic ` - -``` - -### Runtime support - -**Browsers** — all SDK features work natively. `fetch`, `AbortController`, and `ReadableStream` are built into every modern browser; no polyfills are required. - -**Node.js** — every feature, streaming included, works in Node 22 and later (the package's minimum, per `engines.node`). 22 is the only line we test against, and older releases are past end-of-life upstream. Streaming runs on the same `fetch` and `ReadableStream` globals as the rest of the SDK, so it needs **no `EventSource` polyfill** — earlier versions did, and that requirement is gone. - -## Quick Start - - - - -```ts -import { createClient } from '@wavehouse/sdk'; - -const wh = createClient({ - baseURL: 'http://localhost:8080', - auth: async () => getAccessToken(), // omit for public/unauthenticated -}); - -// Query -const { data, error } = await wh.from('clicks').select('page').limit(10); - -// Insert -await wh.from('clicks').insert({ page: '/home', button: 'signup' }); - -// Stream -const stream = wh.from('clicks').stream(); -const unsub = stream.subscribe({ - next: (event) => console.log(event.data), - status: (s) => console.log('Stream:', s), -}); -``` - - - - -```ts -import { createClient } from '@wavehouse/sdk'; - -const wh = createClient({ - baseURL: 'http://localhost:8080', - auth: async () => getAccessToken(), // omit for public/unauthenticated -}); - -// Query -const { data, error } = await wh.from('clicks').select('page').limit(10); - -// Insert -await wh.from('clicks').insert({ page: '/home', button: 'signup' }); - -// Stream -const stream = wh.from('clicks').stream(); -const unsub = stream.subscribe({ - next: (event) => console.log(event.data), - status: (s) => console.log('Stream:', s), -}); -``` - - - - -```ts -import { createClient } from '@wavehouse/sdk'; - -const wh = createClient({ - baseURL: 'http://localhost:8080', - auth: async () => getAccessToken(), // omit for public/unauthenticated -}); - -// Query -const { data, error } = await wh.from('clicks').select('page').limit(10); - -// Insert -await wh.from('clicks').insert({ page: '/home', button: 'signup' }); - -// Stream -const stream = wh.from('clicks').stream(); -const unsub = stream.subscribe({ - next: (event) => console.log(event.data), - status: (s) => console.log('Stream:', s), -}); -``` - - - - -```ts -import { createClient } from '@wavehouse/sdk'; - -const wh = createClient({ - baseURL: 'http://localhost:8080', - auth: async () => getAccessToken(), // omit for public/unauthenticated -}); - -// Query -const { data, error } = await wh.from('clicks').select('page').limit(10); - -// Insert -await wh.from('clicks').insert({ page: '/home', button: 'signup' }); +## First query -// Stream -const stream = wh.from('clicks').stream(); -const unsub = stream.subscribe({ - next: (event) => console.log(event.data), - status: (s) => console.log('Stream:', s), -}); -``` - - - + + ```ts import { createClient } from '@wavehouse/sdk'; -const wh = createClient({ - baseURL: 'http://localhost:8080', - auth: async () => getAccessToken(), // omit for public/unauthenticated -}); +const wh = createClient({ baseURL: 'https://.wavehouse.app' }); -// Query const { data, error } = await wh.from('clicks').select('page').limit(10); - -// Insert -await wh.from('clicks').insert({ page: '/home', button: 'signup' }); - -// Stream -const stream = wh.from('clicks').stream(); -const unsub = stream.subscribe({ - next: (event) => console.log(event.data), - status: (s) => console.log('Stream:', s), -}); +if (error) throw new Error(error.message); +console.log(data); ``` - + -```ts -import { createClient } from 'https://esm.sh/@wavehouse/sdk'; +```go +package main -const wh = createClient({ - baseURL: 'http://localhost:8080', - auth: async () => getAccessToken(), // omit for public/unauthenticated -}); +import ( + "context" + "fmt" + "log" -// Query -const { data, error } = await wh.from('clicks').select('page').limit(10); + wavehouse "github.com/Wave-RF/WaveHouse/clients/go" +) -// Insert -await wh.from('clicks').insert({ page: '/home', button: 'signup' }); +func main() { + wh := wavehouse.NewClient(wavehouse.Config{ + BaseURL: "https://.wavehouse.app", + }) -// Stream -const stream = wh.from('clicks').stream(); -const unsub = stream.subscribe({ - next: (event) => console.log(event.data), - status: (s) => console.log('Stream:', s), -}); + page, err := wh.From("clicks").Select("page").Limit(10).FetchUntyped(context.Background()) + if err != nil { + log.Fatal(err) + } + fmt.Println(page.Data) +} ``` -## Creating a Client - -```ts -import { createClient } from '@wavehouse/sdk'; -import type { Database } from './my-types'; // optional hand-written types - -const wh = createClient({ - baseURL: 'https://.wavehouse.app', - auth: async () => myAuthProvider.getToken(), - options: { - maxRetries: 2, - }, -}); -``` - -### `ClientConfig` - -| Field | Type | Default | Description | -|-------|------|---------|-------------| -| `baseURL` | `string` | — | WaveHouse server URL, optionally including a path prefix (required) | -| `auth` | `() => Promise \| string` | — | Token provider. Omit for public access | -| `options.maxRetries` | `number` | `2` | Retry attempts for failed/5xx **REST** requests; stream reconnects are unbounded ([details](/sdk/streaming#transport-behavior)) | -| `options.headers` | `Record` | — | Headers added to every request ([details](#custom-headers)) | -| `options.fetchOptions` | `RequestInit` | — | Extra `RequestInit` fields merged into every request ([details](#extra-requestinit-fields)) | -| `options.fetch` | `FetchLike` | global `fetch` | HTTP implementation for every request ([details](#supplying-your-own-fetch)) | - -:::note[How the token is transmitted] -The SDK attaches your `auth` token as an `Authorization: Bearer` header on every request, streaming included. It is re-read on each connection attempt, so a stream that outlives its token reconnects with a fresh one instead of replaying an expired credential. - -The token is never placed in the URL. The server still accepts a `?token=` query parameter for clients that cannot set headers — see [API Reference — Authentication](/api#authentication) — but the SDK does not use it, so it never reaches a proxy or CDN access log. -::: - -:::caution[Streaming asks more of a custom `fetch`] -`.stream()` and `.liveQuery()` read the response as it arrives, so an `options.fetch` used with them must return a response with a live streaming body. A response handed back with an absent or already-consumed body fails fast with `SSE_NO_STREAM_BODY`. What the SDK cannot rescue is a wrapper that *awaits* the body before returning — `await res.text()`, or the `res.clone().text()` a logging wrapper reaches for — because on a stream that never ends it never resolves and your function never returns. Both satisfy every REST call, which is what makes the trap easy to walk into. - -`options.headers` is also subject to CORS in a browser: a header outside the safelist joins the preflight, which the origin has to allow. WaveHouse advertises a fixed set, so custom headers reach it cross-origin only when a proxy in front terminates the preflight — the deployment they exist for. Server-side callers never preflight. -::: - -#### Serving under a path prefix - -`baseURL` may carry a path prefix, for a WaveHouse reachable somewhere other than the root of an origin — behind a backend-for-frontend (BFF), an app-server route, or a path-routed ingress. Every request path is appended to it, on both transports: - -```ts -const wh = createClient({ baseURL: 'https://app.example.com/api/wavehouse' }); -// queries → https://app.example.com/api/wavehouse/v1/query -// streams → https://app.example.com/api/wavehouse/v1/stream -``` - -Trailing slashes on `baseURL` are optional, but it must be **absolute** — scheme and host included — and the scheme must be `http` or `https`. A same-origin relative path like `/api/wavehouse` makes every REST call reject with a `TypeError` instead of returning a `Result`, and surfaces on a stream as an `SSE_CONNECT_ERROR` to the subscriber's `error` callback — so build an absolute one: `` `${location.origin}/api/wavehouse` ``. A `ws://`/`wss://` base ends the stream with the same error: streaming is plain HTTP, and there is no WebSocket endpoint. - -The proxy in front must **strip** the prefix before forwarding, since WaveHouse itself always serves at `/v1/…` — see [Behind a reverse proxy → Path prefixes](/reverse-proxy#path-prefixes). - -### Custom headers - -`options.headers` adds headers to every request, streaming included — the usual reason being a header-gated proxy in front of WaveHouse, such as a Cloudflare Access service token: - -```ts -const wh = createClient({ - baseURL: 'https://.wavehouse.app', - options: { - headers: { - 'CF-Access-Client-Id': process.env.CF_ACCESS_CLIENT_ID!, - 'CF-Access-Client-Secret': process.env.CF_ACCESS_CLIENT_SECRET!, - }, - }, -}); -``` - -Header names are matched **case-insensitively**, as HTTP requires, so `authorization` and `Authorization` are the same header rather than two. - -Your headers are applied *underneath* the SDK's own, and a collision means yours is dropped rather than merged: - -- **`auth` keeps `Authorization`.** Setting it here won't displace your token provider. Use `auth` for credentials. -- **`Content-Type` and `Accept` belong to the request.** The SDK knows what it's sending; a global `Content-Type` that outranked it would break requests whose body isn't JSON. - -Nothing is ever comma-joined: on a collision the SDK's value stands alone, and two of your own entries differing only in case collapse to the last one — a header joined rather than replaced is how you end up sending `Content-Type: application/json, image/png`. - -From a **browser**, a cross-origin custom header must also survive CORS preflight, and WaveHouse allow-lists a fixed set (`Accept`, `Authorization`, `Content-Type`, `Last-Event-ID`, `X-Request-ID`) with no config knob. So custom headers work server-side, or from a browser when the proxy in front owns CORS — which is the same proxy the header is usually for. - -Headers are static. For a credential that rotates per request, wrap the transport with [`options.fetch`](#supplying-your-own-fetch); a callback form is tracked in [#459](https://github.com/Wave-RF/WaveHouse/issues/459). - -### Extra `RequestInit` fields - -`options.fetchOptions` is merged into the `RequestInit` of every request, streaming included — for settings that aren't headers and don't warrant replacing the whole transport: - -```ts -const wh = createClient({ - baseURL: 'https://wavehouse.example.com', - options: { fetchOptions: { cache: 'no-store', mode: 'cors' } }, -}); -``` - -It carries any `RequestInit` field — `mode`, `cache`, `keepalive`, `credentials`, `redirect` — plus runtime-specific extensions such as Next.js's `next: { tags }`, which tags the cached response for on-demand revalidation. Only those runtime extensions need a cast; the standard fields are declared on `RequestInit` already. - -:::caution[Two of those fields don't apply to streams] -On `.stream()` and `.liveQuery()`'s live connection the transport keeps `cache` and `redirect` for itself, because there they're correctness rather than preference. It constrains one more without owning it: `credentials` stays yours, forwarded in a browser and dropped elsewhere because some runtimes throw if it is set at all. See [Supplying your own fetch](#supplying-your-own-fetch) for what each is set to and why. - -Everything else reaches streams normally — including `mode` from the example above. Only `cache` is different there: on a stream it is already `no-store`. -::: - -:::note[`credentials: 'include'` needs a proxy that owns CORS] -Sending cookies cross-origin is a common reason to reach for this, but it can't work against WaveHouse's own CORS: it deliberately never emits `Access-Control-Allow-Credentials` (it's a Bearer-token API), and the default `cors_allowed_origins: "*"` makes `include` a hard browser failure regardless. It applies where a fronting proxy answers CORS itself — see [Behind a reverse proxy](/reverse-proxy#header-and-auth-forwarding). Same-origin deployments already send cookies without it. -::: +## Setup -The fields the SDK controls — `method`, `headers`, `body`, and `signal` — always win, so this can't corrupt the request itself. In particular `headers` here is ignored; use `options.headers`, which merges properly. +Installation, client construction, auth, and the error model are the only language-specific parts. [`/sdk/setup`](/sdk/setup) has the package name, install command, and version floor for each client. -### Supplying your own `fetch` - -`options.fetch` replaces the HTTP implementation the SDK uses. Reach for it to route through a proxy, attach client certificates, wrap requests in your own middleware (logging, tracing, circuit breaking), stub HTTP in your own tests without monkey-patching a global, or to work around transport behavior of the runtime you happen to be on: - -```ts -const wh = createClient({ - baseURL: 'https://.wavehouse.app', - options: { - fetch: async (url, init) => { - const started = performance.now(); - const res = await fetch(url, init); - console.log(res.status, url, `${Math.round(performance.now() - started)}ms`); - return res; - }, - }, -}); -``` - -Retries go through the same function, so middleware sees every attempt. - -The SDK always calls your function with a **string** URL and a plain `RequestInit`. Off the response the **REST path** reads `.ok` and `.headers` always, `.text()` on success, and `.status`, `.statusText` plus `.json()` when the response is not `ok` — so a hand-rolled response object needs all six there. A stub used with `.stream()` or `.liveQuery()` needs a different set: `.ok`, `.status`, `.type` (checked for `opaqueredirect`), `.headers` carrying `content-type: text/event-stream`, and a live, unread `.body` — plus `.statusText` and `.json()` if your stub ever answers a stream with a non-`ok` response, which goes through the same error parsing as REST. `.text()` is never called on a stream. - -**On REST**, if your function rejects, the request becomes a `NETWORK_ERROR` result and is retried with backoff — unless the `AbortSignal` you passed has been aborted, in which case the result is `ABORTED` and nothing is retried. That's decided from the signal rather than from what your function threw, so an implementation that signals abort some other way — `AbortSignal.timeout()` raises a `TimeoutError`, `node-fetch` its own `AbortError` class — is reported the same way. On a stream the same rejection is reported to the subscriber as `SSE_NETWORK_ERROR` and re-dialed — including an `AbortError`, since the stream transport applies the same rule: only an abort *it* raised, via `.close()`, ends the stream. See [Error Handling](/sdk/reference#error-handling). - -The option's type is exported as `FetchLike`. It is written out rather than as `typeof fetch`, which resolves differently depending on whether your TypeScript `lib` includes DOM. Its URL parameter is `string` — all the SDK ever passes — which, because function parameters are contravariant, accepts *more* implementations than the wider spelling would: the global `fetch` fits, and so does hand-written `(url: string, init?: RequestInit) => Promise` middleware. Let TypeScript infer your parameters (`(url, init) => …`) and it fits without annotation. - -Streaming goes through your function too. That comes with the extra requirement described above — the response must carry a live streaming body — and the streaming transport keeps ownership of two `RequestInit` fields where the value is load-bearing rather than a preference: `cache` (as an init field; a `Cache-Control` header would fail cross-origin preflight) and `redirect`. It constrains one more without owning it: `credentials` stays yours, forwarded in a browser and dropped elsewhere, because some runtimes throw if it is set at all. - -`redirect` depends on whether the request carries a credential. With one — an `auth` token or configured `headers` — a redirect is refused and reported as `SSE_REDIRECT`, because a cross-origin hop strips `Authorization` (which this endpoint answers with a silently reduced view rather than an error) while forwarding your other headers to wherever the redirect points. Without either, redirects are followed normally, so CDN canonicalization, geo/LB indirection, and an http→https upgrade all work. The test is that concrete pair rather than "is this request authenticated", and **cookies are not part of it**. A cookie isn't stripped the way `Authorization` is — it is re-derived from the cookie store at each hop, so a redirect carries it only while the hop stays inside **both** the request's origin and the cookie's own `Path`. Leaving either loses it. The origin half is the one no cookie attribute can override: under the default `same-origin` credentials mode the browser attaches nothing once a redirect goes cross-origin, whatever the `Domain` — so `app.example.com` → `api.example.com` on a shared `Domain=example.com` session cookie loses it, and so does an http→https upgrade, which is cross-origin by scheme and port. A same-origin rewrite from `/v1/stream` to `/stream` under a `Path=/v1` cookie loses it the other way. This endpoint answers a cookieless request with a reduced view rather than an error ([#478](https://github.com/Wave-RF/WaveHouse/issues/478)). Going cross-origin with `credentials: "include"` usually fails earlier and louder instead: WaveHouse never sends `Access-Control-Allow-Credentials`, so against it the browser blocks the response outright. Behind a proxy that owns CORS and does allow credentials the hop succeeds — and then ordinary cookie scoping decides. `include` *replaces* the origin rule rather than lifting it: the cookie crosses only if its own `Domain` covers the target, so a host-only cookie — what you get with no `Domain` attribute — still stops at its host. If you need to follow a credentialed redirect anyway, supply an `options.fetch` that overrides `redirect`. A `200` that isn't `text/event-stream` is refused too (`SSE_BAD_CONTENT_TYPE`), so an auth gateway's login page fails loudly instead of leaving the stream connected and permanently silent. - -#### Swapping in undici - -The motivating case is a transport bug in the runtime's own HTTP stack, which you can't fix from inside the SDK — for example [undici #5600](https://github.com/nodejs/undici/issues/5600): reusing a keep-alive socket while the event loop is idle stalls the request before it goes out. How bad it gets varies with the runtime and the idle gap: the upstream report measured ~450–465 ms against a 10 ms server, and we have measured anything from ~100 ms to tens of seconds on a server answering instantly. It affects undici 8.8.0–8.9.0, and Node 26 bundles 8.9.0. - -**Upgrading undici is the actual fix** — it landed in 8.10.0. What `options.fetch` buys you is a way to get there without waiting for a new runtime: install undici yourself and route requests through it. The one non-obvious part is that you must pass its dispatcher **explicitly**. - -```ts -import { createClient } from '@wavehouse/sdk'; -import { Agent, fetch as undiciFetch } from 'undici'; // npm install undici — 8.10.0+ - -// Explicit, not implied — see below. -const dispatcher = new Agent(); - -const wh = createClient({ - baseURL: 'https://.wavehouse.app', - options: { - fetch: (url, init) => - undiciFetch(url, { ...init, dispatcher } as never) as unknown as Promise, - }, -}); -``` - -:::caution[Importing a fixed undici is not enough on its own] -undici keeps its connection pool on a shared `globalThis` symbol, and whichever copy loads first claims it. Node claims it for the *bundled* copy the first time anything touches one of its web globals — a `fetch()` call, but equally a `new Headers()` or `new Response()` — not at startup. So which undici owns the pool comes down to a load order you don't really control, and auditing your own code for `fetch` calls won't tell you: any dependency can claim it first, and so can this SDK, which constructs a `Headers` on its abort and retry-exhausted paths. Call `undiciFetch` without a `dispatcher` and it resolves whatever is on that symbol, so requests can go through the buggy pool even though you imported a fixed undici. - -Measured with 8.9.0 loaded first and 8.10.0's `fetch` doing the request, 1.5 s idle gaps against a 10 ms server: - -```text - per-request ms -no explicit dispatcher 21 1514 1495 583 ← still stalling -explicit new Agent() 17 14 12 13 -``` - -An explicit `dispatcher` wins because the request never consults the shared symbol at all. An explicit `setGlobalDispatcher` call (below) wins the other way round — it overwrites the symbol after the first copy claimed it. -::: - -Both casts are load-bearing, for one underlying reason: undici declares its own request/response types, separate from the ones behind your global `fetch`, so the two aren't structurally assignable. `{ ...init, dispatcher } as never` covers the split on `RequestInit`, which differs on `body` and `headers` (`never` is assignable to either spelling, so one snippet compiles whether or not your `lib` includes DOM); and the return cast handles the `Response` mismatch. The URL needs no cast — `FetchLike` declares it as `string`, which undici's `RequestInfo` accepts. They're safe because of the narrow runtime contract above — a string URL and a plain `RequestInit` in, and only the response members enumerated above read back. `undici` is a dependency you add on top of the SDK's single ~1.4 KB one. - -If you're genuinely pinned to an affected undici, the same snippet fixes it — your pinned version, with a dispatcher that never reuses a keep-alive socket: `new Agent({ pipelining: 0 })`. That costs a fresh connection per request, so prefer upgrading. Note that tuning `keepAliveTimeout` does **not** help: the retirement timer is starved by the same idle event loop that triggers the bug, so the socket is still there to be reused. - -To change the transport process-wide instead of per-client, an installed undici's `setGlobalDispatcher(new Agent())` is picked up by Node's built-in global `fetch`, no `options.fetch` needed. That's an explicit write, so it wins the race described above — but only if the *installed* undici is 8.10.0+, since it hands connection handling to that copy. - -### Type-Safe Tables - -Pass a `Database` type to get autocomplete on table names and row types: - -```ts -interface Database { - clicks: { page: string; button: string; score: number; received_timestamp: string }; - users: { id: string; name: string; email: string }; -} - -const wh = createClient({ baseURL: '...' }); -const clicks = wh.from('clicks'); // ✅ autocomplete -const { data } = await clicks.select('page', 'button').limit(10); -// data is Array<{ page: string; button: string; score: number; received_timestamp: string }> | null -``` - -Generate the `Database` interface from a running server with the [codegen CLI](/sdk/reference#codegen-cli). - -## Result Type - -Every async SDK operation returns `Result` — a discriminated union that never throws for anything the server returns (caller and environment errors do throw — see [Error Handling](/sdk/reference#error-handling)): - -```ts -type Result = - | { ok: true; data: T; error: null; hasMore?: boolean; next?: () => Promise> } - | { ok: false; data: null; error: WaveHouseError } - -interface WaveHouseError { - status: number; // HTTP status (0 for network errors) - code: string; // e.g. 'HTTP_400', 'NETWORK_ERROR', 'ABORTED' - message: string; // Human-readable error message - details?: unknown; // Raw response body - retryable: boolean; // Whether SDK would retry this error -} -``` - -Usage pattern — branch on `result.ok` (or destructure `{ data, error }` if you prefer): - -```ts -const result = await wh.from('clicks').select('page').limit(10); -if (result.ok) { - console.log(result.data); // Row[] — TypeScript knows data is non-null here -} else { - console.error(result.error.message); // never throws -} -``` - -The full error-code table lives in [Error Handling](/sdk/reference#error-handling). + + + -## Explore the SDK +## Topics, both languages diff --git a/docs/src/content/docs/sdk/pipes.md b/docs/src/content/docs/sdk/pipes.md deleted file mode 100644 index 8b41f8af..00000000 --- a/docs/src/content/docs/sdk/pipes.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: "SDK Pipes" -description: "Execute and manage named query pipes with @wavehouse/sdk." ---- - -Named pipes are server-defined, parameterized queries (see [Named Pipes guide](/pipes)). The SDK executes pipes for allowed roles and manages their definitions under the admin role. Examples import from `@wavehouse/sdk` or `https://esm.sh/@wavehouse/sdk` (see [Imports & Runtimes](/sdk#imports--runtimes)). - -## Named Pipes — `wh.pipe(name, params?)` - -Execute a pre-defined named query pipe. Returns a `PipeRef` which is **PromiseLike**. - -```ts -// These are equivalent (PipeRef is PromiseLike): -const { data } = await wh.pipe('top_pages', { start_date: '2026-01-01', limit: 50 }).fetch(); -const { data } = await wh.pipe('top_pages', { start_date: '2026-01-01', limit: 50 }); -``` - -### `.fetch(opts?)` - -Execute and return results. Takes `PipeRequestOptions` — `{ signal }` only, narrower than the `.fetch(opts?)` on a [query builder](/sdk/queries), which also accepts `limit`. Passing a `limit` is a compile error rather than a silent no-op. - -`limit` is typed `never` rather than left out, so the rejection also catches a value passed in a variable — leaving it out would only reject an inline object. That cuts both ways: a value *declared* as `RequestOptions` is rejected whether or not it actually carries a limit, since the type permits one. If you share one options object across calls, type it as `PipeRequestOptions` — the table and query-builder `.fetch()` accept that too — or inline `{ signal }` at the pipe call. - -There is no per-call row cap here: the endpoint binds your `params` as the pipe's parameters, so a limit has to be declared in the pipe's SQL as `{{limit}}` (see [Named Pipes](/pipes)) and passed as `wh.pipe(name, { limit })`, as in the example above. - -### `.stream(opts?)` - -Open a live stream (see [Streaming](/sdk/streaming)). - ---- - -## Pipes Admin — `wh.pipes` - -Manage named query pipes. Requires the admin gate — the admin role (`policy.admin_role`) or the [operator key](/api#authentication). - -```ts -// List all pipes -const { data: pipes } = await wh.pipes.list(); - -// Get a single pipe definition -const { data: pipe } = await wh.pipes.get('top_pages'); - -// Create or update -await wh.pipes.set('top_pages', { - sql: 'SELECT page, count() as views FROM clicks GROUP BY page LIMIT {{limit}}', - parameters: [{ name: 'limit', type: 'number', required: false, default: 100 }], - description: 'Top pages by view count', - allowed_roles: ['viewer', 'admin'], -}); - -// Delete -await wh.pipes.delete('old_pipe'); -``` diff --git a/docs/src/content/docs/sdk/pipes.mdx b/docs/src/content/docs/sdk/pipes.mdx new file mode 100644 index 00000000..a4466ad2 --- /dev/null +++ b/docs/src/content/docs/sdk/pipes.mdx @@ -0,0 +1,140 @@ +--- +title: "SDK Pipes" +description: "Execute and manage named query pipes with the WaveHouse SDKs." +--- + +import { Tabs, TabItem } from "@astrojs/starlight/components"; + +Named pipes are server-defined, parameterized queries (see [Named Pipes guide](/pipes)). Both SDKs execute pipes for allowed roles and manage their definitions under the admin role. + +## Executing a pipe + +Reference a pre-defined named query pipe, binding its parameters. + + + + +`wh.pipe(name, params?)` returns a `PipeRef`, which is **PromiseLike**. + +```ts +// These are equivalent (PipeRef is PromiseLike): +const { data } = await wh.pipe('top_pages', { start_date: '2026-01-01', limit: 50 }).fetch(); +const { data } = await wh.pipe('top_pages', { start_date: '2026-01-01', limit: 50 }); +``` + +`.fetch(opts?)` takes `PipeRequestOptions`: `{ signal }` only, narrower than the `.fetch(opts?)` on a [query builder](/sdk/queries#executing-the-query), which also accepts `limit`. Passing a `limit` is a compile error rather than a silent no-op. + +`limit` is typed `never` rather than left out, so the rejection also catches a value passed in a variable, where leaving it out would only reject an inline object. That cuts both ways: a value *declared* as `RequestOptions` is rejected whether or not it actually carries a limit, since the type permits one. If you share one options object across calls, type it as `PipeRequestOptions` (the table and query-builder `.fetch()` accept that too), or inline `{ signal }` at the pipe call. + + + + +`client.Pipe(name, params)` returns a `*PipeRef`. Unlike the TypeScript SDK's `PromiseLike` `PipeRef`, you execute it explicitly. Pass `nil` for `params` if the pipe takes none, or only needs its server-side defaults. + +`wavehouse.Fetch[Row](ctx, pipeRef)` decodes results into `[]Row`. It is a package-level generic function, since Go has no generic methods (the same pattern as `FetchTyped` for queries and `SQL` for raw SQL). `.FetchUntyped(ctx)` is the non-generic method form, decoding into `[]map[string]any`. + +```go +type TopPage struct { + Page string `json:"page"` + Views int `json:"views"` +} + +rows, err := wavehouse.Fetch[TopPage](ctx, + wh.Pipe("top_pages", map[string]any{"start_date": "2026-01-01", "limit": 50}), +) + +untyped, err := wh.Pipe("top_pages", nil).FetchUntyped(ctx) +``` + + + + +There is no per-call row cap here: the endpoint binds your params as the pipe's parameters, so a limit has to be declared in the pipe's SQL as `{{limit}}` (see [Named Pipes](/pipes)) and passed as a parameter, as in the examples above. + +## Streaming a pipe + +Open a live stream from a pipe ref (see [Streaming](/sdk/streaming)). The pipe's SQL and params are **not** applied: the pipe's *name* is used as a table name, so where a table of that name exists you receive its raw events, and otherwise the stream stays silent rather than erroring. Both SDKs behave this way; both wait on a pipe-aware stream endpoint ([#445](https://github.com/Wave-RF/WaveHouse/issues/445)). + + + + +```ts +const stream = wh.pipe('top_pages').stream(); +``` + + + + +```go +stream := wh.Pipe("top_pages", nil).Stream(nil) +defer stream.Close() +``` + + + + +--- + +## Managing pipe definitions + +Create, read, and delete pipe definitions. These sit behind the [admin gate](/sdk/admin) on `/v1/ops/*`: the admin role (`policy.admin_role`) or the [operator key](/api#authentication). + + + + +```ts +// List all pipes +const { data: pipes } = await wh.pipes.list(); + +// Get a single pipe definition +const { data: pipe } = await wh.pipes.get('top_pages'); + +// Create or update +await wh.pipes.set('top_pages', { + sql: 'SELECT page, count() as views FROM clicks GROUP BY page LIMIT {{limit}}', + parameters: [{ name: 'limit', type: 'number', required: false, default: 100 }], + description: 'Top pages by view count', + allowed_roles: ['viewer', 'admin'], +}); + +// Delete +await wh.pipes.delete('old_pipe'); +``` + + + + +```go +// List all pipes. +pipes, err := wh.Pipes.List(ctx) + +// Get a single pipe definition. +pipe, err := wh.Pipes.Get(ctx, "top_pages") + +// Create or update. +err = wh.Pipes.Set(ctx, "top_pages", wavehouse.PipeDef{ + SQL: "SELECT page, count() as views FROM clicks GROUP BY page LIMIT {{limit}}", + Parameters: []wavehouse.ParamDef{ + {Name: "limit", Type: "number", Required: false, Default: 100}, + }, + Description: "Top pages by view count", + AllowedRoles: []string{"viewer", "admin"}, +}) + +// Delete. +err = wh.Pipes.Delete(ctx, "old_pipe") +``` + +`PipeDef` is `Pipe` minus `Name`, which the methods take as a path argument: + +```go +type PipeDef struct { + SQL string + Parameters []ParamDef + Description string + AllowedRoles []string +} +``` + + + diff --git a/docs/src/content/docs/sdk/queries.md b/docs/src/content/docs/sdk/queries.md deleted file mode 100644 index e4e497cb..00000000 --- a/docs/src/content/docs/sdk/queries.md +++ /dev/null @@ -1,257 +0,0 @@ ---- -title: "SDK Queries" -description: "Tables, the chainable query builder, pagination, and raw SQL in @wavehouse/sdk." ---- - -Reading and writing data with `@wavehouse/sdk`: table references, the chainable query builder, cursor pagination, and the admin-only raw-SQL escape hatch. Every call returns the SDK's [`Result`](/sdk#result-type) — nothing throws for anything the server returns (see [Error Handling](/sdk/reference#error-handling) for the caller and environment errors that do). Examples import from `@wavehouse/sdk` or `https://esm.sh/@wavehouse/sdk` (see [Imports & Runtimes](/sdk#imports--runtimes)). - -## Tables — `wh.from(table)` - -`from()` returns a `TableRef` — a reference to a table. It is **NOT thenable**, so it's safe to pass around or store in a variable without triggering requests. - -```ts -const clicks = wh.from('clicks'); -``` - -### `.fetch(opts?)` - -Shortcut for "select every column", with a default limit of 1000. When an access-control policy restricts your role's columns, the server returns only the columns your role is allowed to read — `.fetch()` is never a way around `deny_columns`/`allow_columns` (see [Access control](/access-control#column-permissions)). - -To paginate, chain an explicit `.orderBy()` — a bare `.fetch()` sends no default order (see [Pagination](#pagination)). Ordering, grouping, or filtering by a column your role can't read is rejected, so a column-restricted role must reference only readable columns in those clauses. - -```ts -const { data, error, hasMore, next } = await clicks.fetch(); -const { data } = await clicks.fetch({ limit: 50, signal: controller.signal }); -``` - -### `.insert(data, opts?)` - -Insert one row or many. A single object is sent as a JSON `POST /v1/ingest?table={table}`. An **array** is serialized to NDJSON (one record per line) and sent as a single `application/x-ndjson` request, so a bad record no longer fails or hides the rest of the batch — per-record outcomes come back in the result. - -```ts -// Single row → { ok: true } (or { ok: true, duplicate: true } when dedup skips it) -const { data, error } = await clicks.insert({ page: '/home', button: 'cta' }); - -// Many rows → one NDJSON request, per-record summary -const { data } = await clicks.insert([ - { page: '/home', button: 'cta' }, - { page: '/about', button: 'nav' }, -]); -// data: { ok, total, succeeded, failed, duplicates, results? } -``` - -For an array insert, `data.ok` is `true` only when every record succeeded (`failed === 0`). Inspect `data.failed` and `data.results` (each `{ index, ok|duplicate|error }`, 1-based `index`) for partial failures — the call's top-level `error` is reserved for whole-request failures (network, `404` unknown table, `403` forbidden, `503` backpressure). An empty array is a no-op and sends no request. The array path sends one request regardless of size; bounded-concurrency chunking of very large arrays is tracked in [#196](https://github.com/Wave-RF/WaveHouse/issues/196). - -> The server itself is format-agnostic: `POST /v1/ingest` also accepts a raw JSON array or a single object directly (the `Content-Type` is only a hint), so non-SDK clients can send whichever shape is convenient. See the [API reference](/api#post-v1ingesttabletable--ingest-data). - -### `.insertNDJSON(source, opts?)` - -Insert pre-formatted NDJSON you already have — a `.ndjson` file, a byte stream, or a string — without first parsing it into objects. Accepts a `string`, `Uint8Array`, `Blob`/`File`, or `ReadableStream`; non-string sources are read fully into memory before sending. Returns the same per-record summary as an array `insert`. - -```ts -// From a string -await clicks.insertNDJSON('{"page":"/a"}\n{"page":"/b"}\n'); - -// From a browser (a File is a Blob) -await clicks.insertNDJSON(fileInput.files[0]); - -// From a Node file (fs.openAsBlob; or read it to a string) -import { openAsBlob } from 'node:fs'; -await clicks.insertNDJSON(await openAsBlob('events.ndjson')); -``` - -### `.schema(opts?)` - -Fetch the table's column definitions from ClickHouse. `.schema()` hits `/v1/ops/schema`, an **admin-only** endpoint: the caller must pass the admin gate — resolve to the policy admin role or present the non-JWT [operator key](/api#authentication) via [`options.headers`](/sdk#custom-headers) — or this returns `403`. - -```ts -const { data } = await clicks.schema(); -// data: { name: 'clicks', columns: [ -// { name: 'page', type: 'String', is_nullable: false, has_default: false }, ... -// ] } -``` - -### `.select(...columns)` - -Start a query builder chain. See [Query Builder](#query-builder). - -```ts -const { data } = await clicks.select('page', 'button').where('page', '=', '/home').limit(10); -``` - -### `.selectAll()` - -Selects **every column your role is allowed to read** — the explicit form of what a bare `.fetch()` does. Mutually exclusive with `.select(...)` and with aggregations (`.count()`, `.sum()`, etc.); the server expands it to your allowed columns (never a raw `SELECT *`) and never bypasses `deny_columns`/`allow_columns`. See [Access control → Column permissions](/access-control#column-permissions). - -```ts -const { data } = await clicks.selectAll().where('country', '=', 'US').limit(10); -``` - -### `.stream(opts?)` - -Open a real-time event subscription. See [Streaming](/sdk/streaming). - -```ts -const stream = clicks.stream({ since: '2026-01-01T00:00:00Z' }); -``` - ---- - -## Query Builder - -Returned by `tableRef.select()`. Immutable — every chain method returns a new `QueryBuilder`. The builder is **PromiseLike**, so `await builder` auto-executes `.fetch()`. - -```ts -// These are equivalent: -const result = await clicks.select('page').limit(10).fetch(); -const result = await clicks.select('page').limit(10); // PromiseLike shortcut -``` - -### Chain Methods - -All methods return a new `QueryBuilder` — the original is unchanged. - -#### `.select(...columns)` - -Append columns to the SELECT clause. A literal `'*'` is the column *named* `*`, not a wildcard — use `.selectAll()` for all columns. - -```ts -const q = clicks.select('page').select('button'); // SELECT page, button -``` - -#### `.selectAll()` - -Select every column your role may read (the all-columns wildcard, expanded server-side to your allowed columns). Mutually exclusive with `.select(...)` and with aggregations (`.count()`, `.sum()`, etc.). - -```ts -const q = clicks.selectAll().where('country', '=', 'US'); -``` - -#### `.where(column, op, value)` - -Add a filter condition. SDK operators are translated to backend format. - -```ts -clicks.select('page').where('score', '>', 10).where('page', 'like', '/home%') -``` - -| SDK Operator | Backend | Description | -|-------------|---------|-------------| -| `'='` | `eq` | Equal | -| `'!='` | `neq` | Not equal | -| `'>'` | `gt` | Greater than | -| `'>='` | `gte` | Greater than or equal | -| `'<'` | `lt` | Less than | -| `'<='` | `lte` | Less than or equal | -| `'in'` | `in` | Value in array | -| `'like'` | `like` | SQL LIKE pattern. Case-**sensitive** on `/v1/query`; the client-side filter used by `.stream()` / `.liveQuery()` matches case-**insensitively** | -| `'not_like'` | — | SQL NOT LIKE — **client-side only** (live-query / stream filtering); the `/v1/query` backend rejects it | - -#### Aggregations - -```ts -clicks.select('page') - .count('*', 'total') // COUNT(*) - .sum('score', 'total_score') // SUM(score) - .avg('score', 'avg_score') // AVG(score) - .min('score', 'min_score') // MIN(score) - .max('score', 'max_score') // MAX(score) - .countDistinct('page', 'unique_pages') - .aggregate('uniqExact', 'user_id', 'unique_users') // custom fn -``` - -Each aggregation method signature: `(column: string, alias?: string)`. `count()` defaults to `column='*'`, `alias='count'`. - -#### `.groupBy(...columns)` - -```ts -clicks.select('page').count().groupBy('page') -``` - -#### `.orderBy(column, dir?)` - -```ts -clicks.select('page').count('*', 'total').orderBy('total', 'desc') -``` - -`dir` defaults to `'asc'`. - -#### `.limit(n)` - -```ts -clicks.select().limit(100) -``` - -If no limit is specified, `QueryBuilder.DEFAULT_LIMIT` (1000) is applied automatically to prevent unbounded result sets. The server also enforces the configured maximum (`query.default_max_rows`, default 10,000 rows). - -#### `.timeRange(column, since, until?)` - -Filter by a time window. `since` and `until` accept RFC3339 timestamps or relative durations (`'1h'`, `'30m'`, `'7d'`, `'2w'` — day and week suffixes expand to hours, so `'7d'` is `'168h'`). - -```ts -clicks.select('page').timeRange('received_timestamp', '1h') -clicks.select('page').timeRange( - 'received_timestamp', '2026-01-01T00:00:00Z', '2026-02-01T00:00:00Z' -) -``` - -#### `.cacheTTL(seconds)` - -Records a desired result-cache TTL on the builder. **Currently client-side state only** — the value is never sent to the server, which derives each result's cache TTL adaptively from query execution time. Wiring it through the wire format is tracked in [#280](https://github.com/Wave-RF/WaveHouse/issues/280). - -```ts -clicks.select('page').count().cacheTTL(300) // not yet honored server-side — see #280 -``` - -### `.fetch(opts?)` - -Execute the query. Returns `Result` with optional pagination. - -```ts -const { data, error, hasMore, next } = await clicks.select('page').limit(50).fetch(); - -if (hasMore && next) { - const page2 = await next(); // cursor-based pagination -} -``` - -**Options** — `RequestOptions`: - -| Field | Type | Description | -|-------|------|-------------| -| `signal` | `AbortSignal` | Cancel the request | -| `limit` | `number` | Override builder limit for this fetch | - -A pipe's `.fetch()` takes the narrower `PipeRequestOptions` instead — see [Pipes](/sdk/pipes#fetchopts). - -### `.stream(opts?)` - -Open a live stream from the builder's table. See [Streaming](/sdk/streaming). - -### Pagination - -When `limit` is set and the result contains at least `limit` rows, `hasMore` is `true`. Cursor-based pagination's `next()` walks an **order column** — it adds a filter on that column using the last row's value — so `next()` is only attached when the query has an explicit `.orderBy()`. With no order column the result still reports `hasMore` honestly, but `next` is `undefined` (there is no deterministic cursor to build) — add an `.orderBy()` to paginate. - -```ts -let result = await clicks.select().orderBy('received_timestamp', 'desc').limit(100).fetch(); - -const allRows = [...result.data!]; -while (result.hasMore && result.next) { - result = await result.next(); - if (result.data) allRows.push(...result.data); -} -``` - ---- - -## Raw SQL — `wh.sql(query, opts?)` - -Execute a raw SQL query. `/v1/ops/query` is admin-only: the caller must resolve to the policy admin role (`admin_role`, `"admin"` by default). A tokenless request falls back to the `default_role`, so it is rejected with `403` on any policy that doesn't deliberately set `default_role` to the admin role (a loudly-warned dev-only setting); an invalid or expired token is rejected with `401`. The SDK has no first-class option for the server's non-JWT [operator key](/api#authentication) — an operator can send its `X-Operator-Key` header via [`options.headers`](/sdk#custom-headers). - -```ts -const { data, error } = await wh.sql('SELECT page, count() FROM clicks GROUP BY page LIMIT 10'); -``` - -:::note[No parameter binding through the SDK] -Positional `?` substitution is not supported, and the SDK has no way to forward ClickHouse-style named params (the `WHERE id = {id:UInt32}` + `param_id=42` query-string combo) — the proxy doesn't forward arbitrary query-string params and `wh.sql()` doesn't expose a hook to add them. Inline literals into the SQL, or — for safe binding from user-supplied input — use the structured query builder (`wh.from(table)…`). -::: diff --git a/docs/src/content/docs/sdk/queries.mdx b/docs/src/content/docs/sdk/queries.mdx new file mode 100644 index 00000000..c8384bce --- /dev/null +++ b/docs/src/content/docs/sdk/queries.mdx @@ -0,0 +1,651 @@ +--- +title: "SDK Queries" +description: "Tables, the chainable query builder, pagination, and raw SQL in the WaveHouse SDKs." +--- + +import { Tabs, TabItem } from "@astrojs/starlight/components"; + +Reading and writing data: table references, the chainable query builder, cursor pagination, and the admin-only raw-SQL escape hatch. Both SDKs speak the same wire format and expose the same surface. Pick your language with the tabs below and the choice follows you across every page. Setup, client construction, and the error model live on the per-language pages: [TypeScript](/sdk/setup/typescript) and [Go](/sdk/setup/go). + +## Tables + +The entry point is a table reference. It performs no request, so it is safe to store in a variable or pass around. + + + + +`wh.from(table)` returns a `TableRef`. It is **NOT thenable**, so holding one triggers nothing. + +```ts +const clicks = wh.from('clicks'); +``` + +Every call returns the SDK's [`Result`](/sdk/setup/typescript#result-type): nothing throws for anything the server returns (see [Error Handling](/sdk/reference#error-handling) for the caller and environment errors that do). + + + + +`client.From(table)` returns a `*TableRef`. + +```go +clicks := wh.From("clicks") +``` + +Every request-response operation takes a `context.Context` first and returns `(T, error)`, with the chainable builder methods and `.Stream(opts)` excepted. See [Error Handling](/sdk/setup/go#error-handling). + + + + +### Fetching every column + +Shortcut for "select every column", with a default limit of 1000. When an access-control policy restricts your role's columns, the server returns only the columns your role is allowed to read. This is never a way around `deny_columns`/`allow_columns` (see [Access control](/access-control#column-permissions)). Ordering, grouping, or filtering by a column your role can't read is rejected, so a column-restricted role must reference only readable columns in those clauses. + +To paginate, chain an explicit order: a bare fetch sends no default order (see [Pagination](#pagination)). + + + + +```ts +const { data, error, hasMore, next } = await clicks.fetch(); +const { data } = await clicks.fetch({ limit: 50, signal: controller.signal }); +``` + + + + +`.Fetch(ctx)` is internally `t.SelectAll().Limit(DefaultLimit).FetchUntyped(ctx)`. There is no options struct as in the TypeScript SDK's `.fetch(opts?)`: to override the limit or paginate, chain `.SelectAll().Limit(n).OrderBy(...)` yourself ([Query Builder](#query-builder)). + +```go +page, err := clicks.Fetch(ctx) +if err != nil { + log.Fatal(err) +} +for _, row := range page.Data { + fmt.Println(row["page"]) +} +``` + + + + +### Inserting rows + +Insert one row or many. A single object is sent as a JSON `POST /v1/ingest?table={table}`. A **slice/array** is serialized to NDJSON (one record per line) and sent as a single `application/x-ndjson` request, so a bad record no longer fails or hides the rest of the batch: per-record outcomes come back in the result. + +For a batch, the result is `ok` only when every record succeeded (no records failed). Inspect the failure count and the per-record results (each carrying a **1-based** index) for partial failures. The call's top-level error is reserved for whole-request failures (network, `404` unknown table, `403` forbidden, `503` backpressure). An empty batch is a no-op and sends no request. The batch path sends one request regardless of size; bounded-concurrency chunking of very large batches is tracked in [#196](https://github.com/Wave-RF/WaveHouse/issues/196). + + + + +```ts +// Single row → { ok: true } (or { ok: true, duplicate: true } when dedup skips it) +const { data, error } = await clicks.insert({ page: '/home', button: 'cta' }); + +// Many rows → one NDJSON request, per-record summary +const { data } = await clicks.insert([ + { page: '/home', button: 'cta' }, + { page: '/about', button: 'nav' }, +]); +// data: { ok, total, succeeded, failed, duplicates, results? } +``` + +Each entry of `data.results` is `{ index, ok|duplicate|error }`. + + + + +`.Insert(ctx, data)` picks the path from the input type: a **map or struct** (excluding slices and `[]byte`) goes as JSON; **any slice** (`[]map[string]any`, `[]ClickRow`, …) is serialized to NDJSON via reflection. + +```go +// Single row → InsertResult{OK: true} (or Duplicate: &true when dedup skips it) +res, err := clicks.Insert(ctx, map[string]any{"page": "/home", "button": "cta"}) + +// Many rows (map slice) → one NDJSON request, per-record summary +res, err = clicks.Insert(ctx, []map[string]any{ + {"page": "/home", "button": "cta"}, + {"page": "/about", "button": "nav"}, +}) +// res.OK, res.Total, res.Succeeded, res.Failed, res.Duplicates, res.Results + +// Many rows (typed slice): same NDJSON path, via reflection +type ClickRow struct { + Page string `json:"page"` + Button string `json:"button"` +} +res, err = clicks.Insert(ctx, []ClickRow{ + {Page: "/home", Button: "cta"}, + {Page: "/about", Button: "nav"}, +}) +``` + +Each entry of `res.Results` is an `InsertRecordResult{Index, OK, Duplicate, Error}`. + + + + +> The server itself is format-agnostic: `POST /v1/ingest` also accepts a raw JSON array or a single object directly (the `Content-Type` is only a hint), so non-SDK clients can send whichever shape is convenient. See the [API reference](/api#post-v1ingesttabletable--ingest-data). + +### Inserting pre-formatted NDJSON + +Insert NDJSON you already have — a `.ndjson` file, a byte stream, or a string — without first parsing it into objects. Returns the same per-record summary as a batch insert. + + + + +`.insertNDJSON(source, opts?)` accepts a `string`, `Uint8Array`, `Blob`/`File`, or `ReadableStream`; non-string sources are read fully into memory before sending. + +```ts +// From a string +await clicks.insertNDJSON('{"page":"/a"}\n{"page":"/b"}\n'); + +// From a browser (a File is a Blob) +await clicks.insertNDJSON(fileInput.files[0]); + +// From a Node file (fs.openAsBlob; or read it to a string) +import { openAsBlob } from 'node:fs'; +await clicks.insertNDJSON(await openAsBlob('events.ndjson')); +``` + + + + +`.InsertNDJSON(ctx, ndjson)` takes a `string`. + +```go +// From a literal string. +res, err := clicks.InsertNDJSON(ctx, `{"page":"/a"}`+"\n"+`{"page":"/b"}`) + +// From a file on disk. +raw, err := os.ReadFile("events.ndjson") +if err != nil { + log.Fatal(err) +} +res, err = clicks.InsertNDJSON(ctx, string(raw)) +``` + + + + +### Table schema + +Fetch the table's column definitions from ClickHouse. This hits `/v1/ops/schema`, an **admin-only** endpoint: the caller must pass the admin gate by resolving to the policy admin role or presenting the non-JWT [operator key](/api#authentication), or it returns `403`. + + + + +Send the operator key via [`options.headers`](/sdk/setup/typescript#custom-headers). + +```ts +const { data } = await clicks.schema(); +// data: { name: 'clicks', columns: [ +// { name: 'page', type: 'String', is_nullable: false, has_default: false }, ... +// ] } +``` + + + + +Send the operator key via [`ClientOptions.Headers`](/sdk/setup/go#clientoptions). + +```go +schema, err := clicks.Schema(ctx) +// schema.Name == "clicks" +// schema.Columns: []Column{{Name: "page", Type: "String", IsNullable: false, HasDefault: false}, ...} +``` + + + + +### Selecting columns + +Starts a query builder chain. See [Query Builder](#query-builder) for the chainable methods and how to execute it. + + + + +```ts +const { data } = await clicks.select('page', 'button').where('page', '=', '/home').limit(10); +``` + + + + +```go +page, err := clicks.Select("page", "button"). + Where("page", wavehouse.OpEq, "/home"). + Limit(10). + FetchUntyped(ctx) +``` + + + + +### Selecting every column explicitly + +`selectAll` selects **every column your role is allowed to read**: the explicit form of what a bare fetch does. Mutually exclusive with an explicit column list and with aggregations; for a column-restricted role the server expands it to exactly that role's allowed columns rather than a bare `SELECT *` (unrestricted/admin roles do get `SELECT *`) and never bypasses `deny_columns`/`allow_columns`. See [Access control → Column permissions](/access-control#column-permissions). + + + + +```ts +const { data } = await clicks.selectAll().where('country', '=', 'US').limit(10); +``` + + + + +```go +page, err := clicks.SelectAll().Where("country", wavehouse.OpEq, "US").Limit(10).FetchUntyped(ctx) +``` + + + + +### Opening a stream + +Open a real-time event subscription on the table. See [Streaming](/sdk/streaming). + + + + +```ts +const stream = clicks.stream({ since: '2026-01-01T00:00:00Z' }); +``` + + + + +```go +stream := clicks.Stream(&wavehouse.StreamOptions{Since: "2026-01-01T00:00:00Z"}) +defer stream.Close() +``` + + + + +--- + +## Query Builder + +Returned by the table ref's select methods. Immutable: every chain method returns a new builder, leaving the original unchanged. + + + + +The builder is **PromiseLike**, so `await builder` auto-executes `.fetch()`. + +```ts +// These are equivalent: +const result = await clicks.select('page').limit(10).fetch(); +const result = await clicks.select('page').limit(10); // PromiseLike shortcut +``` + + + + +Go builders do not auto-execute; call `.FetchUntyped(ctx)` or `wavehouse.FetchTyped[Row](ctx, builder)` explicitly. + +```go +page, err := clicks.Select("page").Limit(10).FetchUntyped(ctx) +``` + + + + +### Appending columns + +Append columns to the SELECT clause. A literal `*` is the column *named* `*`, not a wildcard. Use the `selectAll` form for all columns. + + + + +```ts +const q = clicks.select('page').select('button'); // SELECT page, button +``` + + + + +```go +q := clicks.Select("page").Select("button") // SELECT page, button +``` + + + + +### Filtering + +Add a filter condition. Operators are translated to the backend wire format. + + + + +```ts +clicks.select('page').where('score', '>', 10).where('page', 'like', '/home%') +``` + +| SDK Operator | Backend | Description | +|-------------|---------|-------------| +| `'='` | `eq` | Equal | +| `'!='` | `neq` | Not equal | +| `'>'` | `gt` | Greater than | +| `'>='` | `gte` | Greater than or equal | +| `'<'` | `lt` | Less than | +| `'<='` | `lte` | Less than or equal | +| `'in'` | `in` | Value in array | +| `'like'` | `like` | SQL LIKE pattern. Case-**sensitive** on `/v1/query`; the client-side filter used by `.stream()` / `.liveQuery()` matches case-**insensitively** | +| `'not_like'` | `not_like` | SQL NOT LIKE, **client-side only** (live-query / stream filtering); the `/v1/query` backend rejects it | + + + + +```go +clicks.Select("page"). + Where("score", wavehouse.OpGt, 10). + Where("page", wavehouse.OpLike, "/home%") +``` + +| `FilterOp` constant | Backend wire token | Description | +|----------------------|---------------------|--------------| +| `wavehouse.OpEq` | `eq` | Equal | +| `wavehouse.OpNeq` | `neq` | Not equal | +| `wavehouse.OpGt` | `gt` | Greater than | +| `wavehouse.OpGte` | `gte` | Greater than or equal | +| `wavehouse.OpLt` | `lt` | Less than | +| `wavehouse.OpLte` | `lte` | Less than or equal | +| `wavehouse.OpIn` | `in` | Value in array (accepts any Go slice) | +| `wavehouse.OpLike` | `like` | SQL LIKE pattern. Case-**sensitive** on `/v1/query`; the client-side filter used by `.Stream()` / `.LiveQuery()` matches case-**insensitively** | +| `wavehouse.OpNotLike` | `not_like` | SQL NOT LIKE, **client-side only**; `/v1/query` rejects this token | + + + + +### Aggregations + +Custom function names pass through the generic `aggregate` method but are validated server-side against a fixed allowlist (matched case-insensitively): `count`, `sum`, `avg`, `min`, `max`, `countDistinct`, `uniq`, `uniqExact`, `any`, `anyLast`, `argMin`, `argMax`, `groupArray`, `median`, `quantile`, `stddevPop`, `stddevSamp`, `varPop`, `varSamp`. Anything else is rejected with `400 unsupported aggregation function`. + + + + +```ts +clicks.select('page') + .count('*', 'total') // COUNT(*) + .sum('score', 'total_score') // SUM(score) + .avg('score', 'avg_score') // AVG(score) + .min('score', 'min_score') // MIN(score) + .max('score', 'max_score') // MAX(score) + .countDistinct('page', 'unique_pages') + .aggregate('uniqExact', 'user_id', 'unique_users') // allowlisted fn +``` + +Each aggregation method signature: `(column: string, alias?: string)`. `count()` defaults to `column='*'`, `alias='count'`. + + + + +```go +clicks.Select("page"). + Count("*", "total"). // COUNT(*) + Sum("score", "total_score"). // SUM(score) + Avg("score", "avg_score"). // AVG(score) + Min("score", "min_score"). // MIN(score) + Max("score", "max_score"). // MAX(score) + CountDistinct("page", "unique_pages"). + Aggregate("uniqExact", "user_id", "unique_users") // allowlisted fn +``` + +`Count`/`Sum`/`Avg`/`Min`/`Max`/`CountDistinct` take `(column, alias)`; `.Aggregate` takes `(fn, column, alias)`. With an empty alias, `Count` defaults to `count` (and `column=""` becomes `*`), `Sum`/`Avg`/`Min`/`Max` to `sum_`/`avg_`/`min_`/`max_`, and `CountDistinct` to `count_distinct_`; `Aggregate` has no default and sends `""`. + + + + +### Grouping and ordering + +Order direction defaults to ascending. + + + + +```ts +clicks.select('page').count().groupBy('page') +clicks.select('page').count('*', 'total').orderBy('total', 'desc') +``` + + + + +```go +clicks.Select("page").Count("", "").GroupBy("page") +clicks.Select("page").Count("", "total").OrderBy("total", "desc") +``` + +`OrderBy`'s `dir` defaults to `"asc"` when passed `""`. + + + + +### Limiting + +If no limit is specified, the SDK's default limit (1000) is applied automatically to prevent unbounded result sets. The server also enforces the configured maximum (`query.default_max_rows`, default 10,000 rows). + + + + +```ts +clicks.select().limit(100) // QueryBuilder.DEFAULT_LIMIT otherwise +``` + + + + +```go +clicks.Select().Limit(100) // wavehouse.DefaultLimit otherwise +``` + + + + +### Time ranges + +Filter by a time window. Both bounds accept RFC3339 timestamps or relative durations (`1h`, `30m`, `7d`, `2w`; day and week suffixes expand to hours, so `7d` is `168h`). + + + + +`until` is optional; omit it for an open-ended range. + +```ts +clicks.select('page').timeRange('received_timestamp', '1h') +clicks.select('page').timeRange( + 'received_timestamp', '2026-01-01T00:00:00Z', '2026-02-01T00:00:00Z' +) +``` + + + + +Pass `""` for `until` for an open-ended range. + +```go +clicks.Select("page").TimeRange("received_timestamp", "1h", "") +clicks.Select("page").TimeRange( + "received_timestamp", "2026-01-01T00:00:00Z", "2026-02-01T00:00:00Z", +) +``` + + + + +### Cache TTL + +Records a desired result-cache TTL on the builder. **Currently client-side state only**: the value is never sent to the server, which derives each result's cache TTL adaptively from query execution time. Wiring it through the wire format is tracked in [#280](https://github.com/Wave-RF/WaveHouse/issues/280). + + + + +```ts +clicks.select('page').count().cacheTTL(300) // not yet honored server-side, see #280 +``` + + + + +```go +clicks.Select("page").Count("", "").CacheTTL(300) // not yet honored server-side, see #280 +``` + + + + +### Executing the query + + + + +`.fetch(opts?)` returns `Result` with optional pagination. + +```ts +const { data, error, hasMore, next } = await clicks.select('page').limit(50).fetch(); + +if (hasMore && next) { + const page2 = await next(); // cursor-based pagination +} +``` + +**Options** (`RequestOptions`): + +| Field | Type | Description | +|-------|------|-------------| +| `signal` | `AbortSignal` | Cancel the request | +| `limit` | `number` | Override builder limit for this fetch | + +A pipe's `.fetch()` takes the narrower `PipeRequestOptions` instead. See [Pipes](/sdk/pipes#executing-a-pipe). + + + + +`wavehouse.FetchTyped[Row](ctx, q)` decodes rows into `[]Row`; `.FetchUntyped(ctx)` decodes into `[]map[string]any`. Both return a `Page[T]`. Typed fetching is a package-level generic function because Go has no generic methods. + +```go +type PageCount struct { + Page string `json:"page"` + Count int `json:"total"` +} + +page, err := wavehouse.FetchTyped[PageCount](ctx, + clicks.Select("page").Count("*", "total").GroupBy("page"), +) +// page.Data is []PageCount + +untyped, err := clicks.Select("page").OrderBy("page", "asc").Limit(50).FetchUntyped(ctx) +``` + + + + +### Streaming from a builder + +A builder can also open a stream instead of fetching: its filters and column projection are then applied client-side to the live events, which is the only place that client-side filtering happens. See [Streaming → Client-side stream filtering](/sdk/streaming#client-side-stream-filtering). + +### Pagination + +When a limit is set and the result contains at least that many rows, the result reports that more rows are available. Cursor-based pagination walks an **order column** (it adds a filter on that column using the last row's value), so the `next` cursor is only attached when the query has an explicit order. With no order column the result still reports "has more" honestly, but there is no cursor to build. + +The cursor filter is strict (`gt`/`lt` against the last row's value) and uses only that one order column, with no tie-breaker, so rows sharing the boundary value with the last row of a page are skipped. Paginate on a column that is unique per row (or made unique by a monotonic timestamp), or accept that ties at a page edge can be dropped. Both SDKs share this limitation ([#452](https://github.com/Wave-RF/WaveHouse/issues/452)). + + + + +```ts +let result = await clicks.select().orderBy('received_timestamp', 'desc').limit(100).fetch(); + +const allRows = [...result.data!]; +while (result.hasMore && result.next) { + result = await result.next(); + if (result.data) allRows.push(...result.data); +} +``` + +Rows decode with JSON numbers as JS `number`s, so an integer cursor column past `Number.MAX_SAFE_INTEGER` (2^53) loses exactness and pagination can repeat or skip a row at that scale. The Go SDK's `FetchTyped` with an `int64` field avoids this; there is no JS equivalent short of a string or `bigint` column. + + + + +Both fetch methods return a `Page[T]`: + +```go +type Page[T any] struct { + Data []T + HasMore bool + Next func(ctx context.Context) (*Page[T], error) // nil when no cursor is available +} +``` + +`Next` walks the **first** `.OrderBy()` column. Without one it is `nil`, and if the order column is missing from `.Select(...)` it returns an empty page. + +```go +page, err := clicks.Select(). + OrderBy("received_timestamp", "desc"). + Limit(100). + FetchUntyped(ctx) +if err != nil { + log.Fatal(err) +} + +allRows := append([]map[string]any(nil), page.Data...) +for page.HasMore && page.Next != nil { + page, err = page.Next(ctx) + if err != nil { + log.Fatal(err) + } + allRows = append(allRows, page.Data...) +} +``` + +On the untyped path (`FetchUntyped` / `TableRef.Fetch`), JSON numbers decode as `float64`, so integer cursors lose exactness past 2^53 and pagination can repeat or skip a row; `FetchTyped` with an `int64` field keeps it exact. + + + + +--- + +## Raw SQL + +Execute a raw SQL query. `/v1/ops/query` is admin-only: the caller must resolve to the policy admin role (`admin_role`, `"admin"` by default). A tokenless request falls back to the `default_role`, so it is rejected with `403` on any policy that doesn't deliberately set `default_role` to the admin role (a loudly-warned dev-only setting); an invalid or expired token is rejected with `401`. + + + + +The SDK has no first-class option for the server's non-JWT [operator key](/api#authentication): an operator can send its `X-Operator-Key` header via [`options.headers`](/sdk/setup/typescript#custom-headers). + +```ts +const { data, error } = await wh.sql('SELECT page, count() FROM clicks GROUP BY page LIMIT 10'); +``` + + + + +An operator key authorizes `/v1/ops/*` as well; send it via [`ClientOptions.Headers`](/sdk/setup/go#clientoptions). Use `map[string]any` for dynamic schemas. + +```go +rows, err := wavehouse.SQL[map[string]any](ctx, wh, + "SELECT page, count() AS total FROM clicks GROUP BY page LIMIT 10") + +// Or decode into a struct that matches the projected columns/aliases. +// NOTE: this path forwards ClickHouse's own JSON, which QUOTES 64-bit +// integers (count() is UInt64), so decode them with the `,string` tag, or +// use map[string]any. See Reference → Codegen CLI for the full story. +type PageTotal struct { + Page string `json:"page"` + Total uint64 `json:"total,string"` +} +typed, err := wavehouse.SQL[PageTotal](ctx, wh, + "SELECT page, count() AS total FROM clicks GROUP BY page LIMIT 10") +``` + + + + +:::note[No parameter binding through the SDK] +Positional `?` substitution is not supported, and neither SDK can forward ClickHouse-style named params (the `WHERE id = {id:UInt32}` + `param_id=42` query-string combo): the proxy doesn't forward arbitrary query-string params and the raw-SQL entry point exposes no hook to add them. Inline literals into the SQL, or — for safe binding from user-supplied input — use the structured query builder. +::: diff --git a/docs/src/content/docs/sdk/reference.md b/docs/src/content/docs/sdk/reference.md deleted file mode 100644 index bd511700..00000000 --- a/docs/src/content/docs/sdk/reference.md +++ /dev/null @@ -1,188 +0,0 @@ ---- -title: "SDK Reference & CLI" -description: "Error codes, AbortController, the full API tree, the codegen CLI, and E2E testing with @wavehouse/sdk." ---- - -Cross-cutting reference for `@wavehouse/sdk`: cancellation, the error model behind every [`Result`](/sdk#result-type), the complete API tree at a glance, and the tooling that ships in the package. - -## AbortController Support - -All async operations accept an `AbortSignal` for cancellation: - -```ts -const controller = new AbortController(); -setTimeout(() => controller.abort(), 5000); // 5s timeout - -const { data, error } = await wh.from('clicks').fetch({ signal: controller.signal }); -if (error?.code === 'ABORTED') { - console.log('Request timed out'); -} -``` - ---- - -## Error Handling - -The SDK **never throws** for anything the server returns — all API errors come back in `Result.error`. It does throw on caller and environment errors: a non-absolute `baseURL` (REST calls reject with a `TypeError`; streams report `SSE_CONNECT_ERROR` to the subscriber's `error` callback — see [Serving under a path prefix](/sdk#serving-under-a-path-prefix)), `.stream()` / `.liveQuery()` in a runtime with no global `fetch` and no `options.fetch` (see [Runtime support](/sdk#runtime-support)), and an `auth` callback that rejects — a token-refresh failure propagates out of the REST call, and on a stream is reported as a retryable `SSE_AUTH_ERROR`. One more exception escapes an SDK call synchronously, though it is yours rather than ours: your own `status` handler throwing on the first `.subscribe()` or `.liveQuery()`, described under *If your own callback throws* below. - -| Status | Code | Retryable | Description | -|--------|------|-----------|-------------| -| 400 | `HTTP_400` | No | Bad request (validation, missing fields) | -| 401 | `HTTP_401` | No | On REST, a present-but-invalid or expired JWT that a gate then denied. **WaveHouse itself** never returns `401` for a *missing* token — that resolves to `default_role`, and a denial is `403`. On a stream it is always from something in front, since `/v1/stream` is ungated | -| 403 | `HTTP_403` | No | Insufficient permissions | -| 404 | `HTTP_404` | No | Table or pipe not found | -| 500 | `HTTP_500` | Yes | Server error (retried per `maxRetries`) | -| 503 | `HTTP_503` | Yes | Service unavailable (auto-retries with `Retry-After`) | -| 0 | `NETWORK_ERROR` | Yes | Network failure (retried with exponential backoff) | -| 0 | `ABORTED` | No | Request canceled via `AbortSignal` | -| 0 | `SSE_CONNECT_ERROR` | No | Stream could not be started (e.g. a non-absolute `baseURL`) | -| 0 | `SSE_AUTH_ERROR` | Yes | The `auth` callback threw while minting a token for an attempt | -| 0 | `SSE_NETWORK_ERROR` | Yes | Stream request failed to reach the server | -| 0 | `SSE_READ_ERROR` | Yes | Stream was interrupted mid-read | -| 0 | `SSE_PARSE_ERROR` | Yes (usually nothing to re-dial) | Unparseable frame — reported and skipped, the connection continues. The buffer cap reports here too, then ends the connection (see below) | -| *(response status)* | `SSE_NO_STREAM_BODY` | No | A configured `options.fetch` returned a response with no readable body | -| *(0 in a browser, the 3xx in Node)* | `SSE_REDIRECT` | No | The stream endpoint redirected; point `baseURL` at the final URL | -| *(response status)* | `SSE_BAD_CONTENT_TYPE` | No | A `200` that wasn't `text/event-stream` — usually a gateway's login page | -| *(response status)* | `HTTP_4xx` / `HTTP_5xx` | Per status | The stream request was rejected — same codes as REST | - -The `SSE_*` codes arrive on the subscriber's `error` callback rather than in a `Result.error`, since a stream has no single result to carry them. Both transports act on `retryable`; what differs is when the error reaches you. All four combinations, with the two exceptions noted under the table: - -| | Retryable | Not retryable | -|---|---|---| -| **REST** | retried up to `maxRetries`, then returned — the flag records what was already tried | returned on the first attempt, no backoff (every `4xx`) | -| **Stream** | reported *before* the transport re-dials, which it does indefinitely | reported once; the stream closes and stays closed | - -On REST, `ABORTED` is the one error raised *by* a backoff rather than by an attempt: aborting during a retry sleep ends the call immediately with `ABORTED` instead of waiting the delay out. On a stream, `SSE_PARSE_ERROR` is flagged retryable, but an ordinary bad frame is skipped in place — no re-dial, no close. The one that *does* re-dial is the buffer-cap overflow described below. - -On a stream, a retryable failure is re-dialed on a jittered exponential backoff (capped at 30s, and reset only once a connection has held for a few seconds — so a server that accepts and instantly closes still backs off), with the `status` callback moving `reconnecting` → `live`. - -Rejected requests surface the real status and message rather than an opaque connection failure — in a browser going cross-origin, though, only when the rejection passes CORS and the gateway answered whatever preflight the request triggers — `Authorization`, configured `headers`, or `Last-Event-ID` once the stream resumes; a rejected preflight or a response without `Access-Control-Allow-Origin` reaches you as a retryable network error instead — indistinguishable from a drop, and retried. Any `4xx` ends the stream, since repeating the request won't usually talk whatever rejected it round — the exception being a `429` or `408` from a fronting rate limiter, which is transient even though the stream still ends, so catch it and open a new one after a delay ([#469](https://github.com/Wave-RF/WaveHouse/issues/469)). Note that **WaveHouse never rejects a stream for authentication**: `/v1/stream` is ungated, so an expired or missing token resolves to `default_role` and you get a `200` with a filtered view, not a `401`. The one 4xx it raises itself is `400` for a missing or empty `table`, and only on the stream route: a `404` or `405` means the request never reached that route, most often a `baseURL` path prefix your proxy didn't strip. Any other 4xx comes from something in front — an auth gateway, a proxy. That silent-downgrade behavior is exactly why `auth` is re-read on every connection attempt, and [#239](https://github.com/Wave-RF/WaveHouse/issues/239) tracks enforcing expiry server-side. `SSE_CONNECT_ERROR` and `SSE_NO_STREAM_BODY` are configuration faults, so fix the cause and start a new stream. - -`SSE_PARSE_ERROR` is the one code that isn't a connection outcome: it's reported and *skipped*, and the connection keeps reading — one bad frame shouldn't cost you the stream. For an ordinary bad frame its `retryable: true` is therefore vestigial — nothing is re-dialed. Two exceptions, one to each half of that reported-and-skipped rule. A frame whose `data` isn't valid JSON is skipped but never *reported* — `console.warn` and dropped, with no `error` callback. The parser's 16 MiB buffer cap is reported but not *skipped*: an overflow terminates the parser, so the transport stops reading and reconnects rather than feeding it again. - -**If your own callback throws.** For anything delivered *through the transport* — `next`, `status`, or `error` — a throw never ends the stream: it is isolated and logged to the console, and never routed to your `error` callback, so a handler that swallows its own failures fails silently. - -**Wrap your handler bodies in your own `try`/`catch`.** That isolation is a backstop against one bad callback killing the connection, not a promise that throwing is harmless. Four paths sit outside it, all tracked in [#473](https://github.com/Wave-RF/WaveHouse/issues/473): - -- **The first `status` call** — `.subscribe()`'s, and the one `liveQuery()` makes internally on your behalf. It fires synchronously with the current state before the transport is involved, and is unguarded, so the throw propagates back out of the call *you* made, unlogged. Out of `.subscribe()` that leaves your subscriber registered with no unsubscribe function returned. Out of `liveQuery()` it is worse: you get no handle at all, so there is nothing to `.close()`; the backfill never starts, so `initial()` is never called; and the stream opened a moment earlier keeps running — connected, reconnecting on its own — with nothing referencing it. Because the backfill never ran, the buffering phase also never ends — your `next()` is never called at all, and every event the stream receives is appended to a buffer you have no way to reach. Passing `opts.signal` is the only way to stop it. Since this call always happens, it is the *first* thing a throwing `status` handler does. -- **Delivery to other consumers stops at the one that threw** — later subscribers and any concurrent `for await`. For the iterator the event is *dropped*, not queued, because the fan-out runs before the event is handed to a waiter or buffered. And if the throw lands on the terminal `closed` status, the iterator is never marked done either, so a `for await` waits forever against a dead stream until something calls `.close()`. Order-dependent and invisible. -- **A throwing `status` handler subscribed before `.connected()`** makes that promise reject with its timeout error against a stream that is already `live`, because the throw aborts the fan-out before `connected()`'s internal watcher is reached. -- **Inside `liveQuery()`**, a throw from `initial()`, or from `next()` during the backfill flush, is absorbed by the backfill's own error path, with no log and no signal. The cost differs: `initial()` throws *before* the flush starts, so every buffered event is lost, while a `next()` throw mid-flush loses only the remainder. Either way, later live events reach `next()` as normal, with a further throw isolated and logged. - -`SSE_AUTH_ERROR` is the one caller-side failure that isn't terminal. A rejecting `auth` callback propagates out of a REST call, but on a stream — where `auth` is invoked on every connection attempt rather than once per stream — a token endpoint having a bad minute is treated as transient and retried, rather than tearing down a stream that is otherwise healthy. - ---- - -## Full API Tree - -```text -createClient(config) → WaveHouseClient -├── .from(table) → TableRef (NOT thenable) -│ ├── .fetch(opts?) → Promise> -│ ├── .select(...cols?) → QueryBuilder (PromiseLike) -│ │ ├── .select() .selectAll() .where() .count() .sum() .avg() .min() .max() -│ │ │ .countDistinct() .aggregate() .groupBy() .orderBy() -│ │ │ .limit() .timeRange() .cacheTTL() -│ │ ├── .fetch(opts?) → Promise> -│ │ ├── .stream(opts?) → StreamController -│ │ └── .liveQuery(subscriber, opts?) → LiveQuery -│ ├── .selectAll() → QueryBuilder (PromiseLike) -│ ├── .insert(data) → Promise> -│ ├── .insertNDJSON(source) → Promise> -│ ├── .schema() → Promise> (admin) -│ └── .stream(opts?) → StreamController -├── .pipe(name, params?) → PipeRef (PromiseLike) -│ ├── .fetch(opts?) → Promise> // { signal } only — no limit -│ └── .stream(opts?) → StreamController -├── .pipes (admin) -│ ├── .list() → Promise> -│ ├── .get(name) → Promise> -│ ├── .set(name, def) → Promise> -│ └── .delete(name) → Promise> -├── .sql(query, opts?) → Promise> (admin) -├── .schema (admin) -│ ├── .list() → Promise> -│ └── .refresh() → Promise> -├── .policy (admin) -│ ├── .get() → Promise> -│ ├── .set(policy) → Promise> -│ └── .validate(policy) → Promise> -├── .dlq (admin) -│ ├── .list() → Promise> -│ ├── .table(name) → Promise> -│ └── .stream() → StreamController // not yet functional server-side — #197 -└── .sys - └── .health() → Promise> - -StreamController (NOT thenable) -├── .subscribe({ next, status?, error? }) → unsubscribe() -├── .connected(timeoutMs?) → Promise -├── .close() -├── .status → StreamStatus -└── [Symbol.asyncIterator]() → AsyncIterableIterator -``` - -## Codegen CLI - -Generate TypeScript types from a running WaveHouse instance. The package ships a `wavehouse-codegen` bin, so after installing `@wavehouse/sdk` you can run it with `npx`: - -```bash -npx wavehouse-codegen --url http://localhost:8080 --out ./src/db.d.ts - -# Or, working inside this repo (clients/ts/): -pnpm codegen --url http://localhost:8080 --out ./src/db.d.ts -``` - -Codegen reads `/v1/ops/schema`, which is **admin-only**. Against a non-dev server, pass an admin-role token with `--auth ` or the request is denied with `403`. - -**Options:** - -| Flag | Description | Default | -|------|-------------|---------| -| `--url`, `-u` | WaveHouse base URL | `http://localhost:8080` | -| `--out`, `-o` | Output .d.ts file path | `./wavehouse.d.ts` | -| `--auth`, `-a` | Bearer token (if auth required) | — | - -**Example output:** - -```ts -// Auto-generated by @wavehouse/sdk codegen -export interface Database { - clicks: ClicksRow; - events: EventsRow; -} - -export interface ClicksRow { - event_id: string; - page: string; - user_id: string; - duration_ms: number; - received_timestamp: string; -} -``` - -**ClickHouse → TypeScript type mapping:** - -| ClickHouse Type | TypeScript Type | -|----------------|-----------------| -| `String`, `FixedString`, `UUID`, `DateTime*`, `Date*`, `Enum*`, `IPv4/6` | `string` | -| `UInt*`, `Int*`, `Float*`, `Decimal*` | `number` | -| `Bool` | `boolean` | -| `Nullable(T)` | `T \| null` | -| `Array(T)` | `T[]` | -| `Map(K, V)` | `Record` | -| `LowCardinality(T)` | same as `T` | - -## E2E Testing - -The SDK doubles as the E2E integration test harness. Tests in `tests/e2e/sdk/` exercise the full pipeline (ingest → ClickHouse → query) through the SDK, validating both the backend and the client library in one pass. - -```bash -# Run all E2E tests: the orchestrator boots a ClickHouse testcontainer + -# the wavehouse-cov binary, then runs the SDK suite -make test-e2e -``` - -Test files live in `tests/e2e/sdk/` (each `*.test.ts`): `admin`, `auth`, `batching`, `cache`, `dlq`, `ingest`, `ndjson`, `query`, `streaming`, `stress`, plus `helpers` — a stack-free unit test of the harness's own `waitForCondition` poll helper rather than a pipeline test. - -See [Development Guide — E2E Tests via SDK](/development#e2e-tests-via-sdk) for architecture details and workflow tips. diff --git a/docs/src/content/docs/sdk/reference.mdx b/docs/src/content/docs/sdk/reference.mdx new file mode 100644 index 00000000..810ad68d --- /dev/null +++ b/docs/src/content/docs/sdk/reference.mdx @@ -0,0 +1,389 @@ +--- +title: "SDK Reference & CLI" +description: "Error codes, cancellation, the full API tree, and the codegen CLIs for the WaveHouse SDKs." +--- + +import { Tabs, TabItem } from "@astrojs/starlight/components"; + +Cross-cutting reference for both SDKs: cancellation, the error model, the complete API tree at a glance, and the tooling each package ships. + +## Cancellation + + + + +All async operations accept an `AbortSignal`: + +```ts +const controller = new AbortController(); +setTimeout(() => controller.abort(), 5000); // 5s timeout + +const { data, error } = await wh.from('clicks').fetch({ signal: controller.signal }); +if (error?.code === 'ABORTED') { + console.log('Request timed out'); +} +``` + + + + +Non-streaming operations take a `context.Context` as their first argument (the analog of TypeScript's `AbortSignal`). Cancel via a timeout or an explicit `cancel()`; cancellation returns immediately, without retrying, as `&wavehouse.Error{Status: 0, Code: "ABORTED", Retryable: false}`. + +```go +ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) +defer cancel() + +page, err := wh.From("clicks").Fetch(ctx) +var whErr *wavehouse.Error +if errors.As(err, &whErr) && whErr.Code == "ABORTED" { + fmt.Println("Request timed out") +} +``` + +`.Stream(opts)` ignores `context.Context`; the returned `*StreamController` manages its own context and goroutine, closed via `.Close()`. See [Streaming → Stream options](/sdk/streaming#stream-options). + + + + +--- + +## Error Handling + +Neither SDK raises for anything the server returns: API errors are values. Both raise (or return plain wrapped errors) for caller and environment faults instead. + + + + +All API errors come back in [`Result.error`](/sdk/setup/typescript#result-type). The SDK does throw on caller and environment errors: a non-absolute `baseURL` (REST calls reject with a `TypeError`; streams report `SSE_CONNECT_ERROR` to the subscriber's `error` callback; see [Serving under a path prefix](/sdk/setup/typescript#serving-under-a-path-prefix)), `.stream()` / `.liveQuery()` in a runtime with no global `fetch` and no `options.fetch` (see [Runtime support](/sdk/setup/typescript#runtime-support)), and an `auth` callback that rejects: a token-refresh failure propagates out of the REST call, and on a stream is reported as a retryable `SSE_AUTH_ERROR`. One more exception escapes an SDK call synchronously, though it is yours rather than ours: your own `status` handler throwing on the first `.subscribe()` or `.liveQuery()`, described under *If your own callback throws* below. + +| Status | Code | Retryable | Description | +|--------|------|-----------|-------------| +| 400 | `HTTP_400` | No | Bad request (validation, missing fields) | +| 401 | `HTTP_401` | No | On REST, a present-but-invalid or expired JWT that a gate then denied. **WaveHouse itself** never returns `401` for a *missing* token: that resolves to `default_role`, and a denial is `403`. On a stream it is always from something in front, since `/v1/stream` is ungated | +| 403 | `HTTP_403` | No | Insufficient permissions | +| 404 | `HTTP_404` | No | Table or pipe not found | +| 500 | `HTTP_500` | Yes | Server error (retried per `maxRetries`) | +| 503 | `HTTP_503` | Yes | Service unavailable (auto-retries with `Retry-After`) | +| 0 | `NETWORK_ERROR` | Yes | Network failure (retried with exponential backoff) | +| 0 | `ABORTED` | No | Request canceled via `AbortSignal` | +| 0 | `SSE_CONNECT_ERROR` | No | Stream could not be started (e.g. a non-absolute `baseURL`) | +| 0 | `SSE_AUTH_ERROR` | Yes | The `auth` callback threw while minting a token for an attempt | +| 0 | `SSE_NETWORK_ERROR` | Yes | Stream request failed to reach the server | +| 0 | `SSE_READ_ERROR` | Yes | Stream was interrupted mid-read | +| 0 | `SSE_PARSE_ERROR` | Yes (usually nothing to re-dial) | Unparseable frame, reported and skipped; the connection continues. The buffer cap reports here too, then ends the connection (see below) | +| *(response status)* | `SSE_NO_STREAM_BODY` | No | A configured `options.fetch` returned a response with no readable body | +| *(0 in a browser, the 3xx in Node)* | `SSE_REDIRECT` | No | The stream endpoint redirected; point `baseURL` at the final URL | +| *(response status)* | `SSE_BAD_CONTENT_TYPE` | No | A `200` that wasn't `text/event-stream`, usually a gateway's login page | +| *(response status)* | `HTTP_4xx` / `HTTP_5xx` | Per status | The stream request was rejected, same codes as REST | + +The `SSE_*` codes arrive on the subscriber's `error` callback rather than in a `Result.error`, since a stream has no single result to carry them. Both transports act on `retryable`; what differs is when the error reaches you. All four combinations, with the two exceptions noted under the table: + +| | Retryable | Not retryable | +|---|---|---| +| **REST** | retried up to `maxRetries`, then returned; the flag records what was already tried | returned on the first attempt, no backoff (every `4xx`) | +| **Stream** | reported *before* the transport re-dials, which it does indefinitely | reported once; the stream closes and stays closed | + +On REST, `ABORTED` is the one error raised *by* a backoff rather than by an attempt: aborting during a retry sleep ends the call immediately with `ABORTED` instead of waiting the delay out. On a stream, `SSE_PARSE_ERROR` is flagged retryable, but an ordinary bad frame is skipped in place: no re-dial, no close. The one that *does* re-dial is the buffer-cap overflow described below. + +On a stream, a retryable failure is re-dialed on a jittered exponential backoff (capped at 30s, and reset only once a connection has held for a few seconds, so a server that accepts and instantly closes still backs off), with the `status` callback moving `reconnecting` → `live`. + +Rejected requests surface the real status and message rather than an opaque connection failure. In a browser going cross-origin, though, that only holds when the rejection passes CORS and the gateway answered whatever preflight the request triggers (`Authorization`, configured `headers`, or `Last-Event-ID` once the stream resumes); a rejected preflight or a response without `Access-Control-Allow-Origin` reaches you as a retryable network error instead, indistinguishable from a drop, and retried. Any `4xx` ends the stream, since repeating the request won't usually talk whatever rejected it round. The exception is a `429` or `408` from a fronting rate limiter, which is transient even though the stream still ends, so catch it and open a new one after a delay ([#469](https://github.com/Wave-RF/WaveHouse/issues/469)). Note that **WaveHouse never rejects a stream for authentication**: `/v1/stream` is ungated, so an expired or missing token resolves to `default_role` and you get a `200` with a filtered view, not a `401`. The one 4xx it raises itself is `400` for a missing or empty `table`, and only on the stream route: a `404` or `405` means the request never reached that route, most often a `baseURL` path prefix your proxy didn't strip. Any other 4xx comes from something in front: an auth gateway, a proxy. That silent-downgrade behavior is exactly why `auth` is re-read on every connection attempt, and [#239](https://github.com/Wave-RF/WaveHouse/issues/239) tracks enforcing expiry server-side. `SSE_CONNECT_ERROR` and `SSE_NO_STREAM_BODY` are configuration faults, so fix the cause and start a new stream. + +`SSE_PARSE_ERROR` is the one code that isn't a connection outcome: it's reported and *skipped*, and the connection keeps reading: one bad frame shouldn't cost you the stream. For an ordinary bad frame its `retryable: true` is therefore vestigial: nothing is re-dialed. Two exceptions, one to each half of that reported-and-skipped rule. A frame whose `data` isn't valid JSON is skipped but never *reported*: `console.warn` and dropped, with no `error` callback. The parser's 16 MiB buffer cap is reported but not *skipped*: an overflow terminates the parser, so the transport stops reading and reconnects rather than feeding it again. + +**If your own callback throws.** For anything delivered *through the transport* — `next`, `status`, or `error` — a throw never ends the stream: it is isolated and logged to the console, and never routed to your `error` callback, so a handler that swallows its own failures fails silently. + +**Wrap your handler bodies in your own `try`/`catch`.** That isolation is a backstop against one bad callback killing the connection, not a promise that throwing is harmless. Four paths sit outside it, all tracked in [#473](https://github.com/Wave-RF/WaveHouse/issues/473): + +- **The first `status` call**: `.subscribe()`'s, and the one `liveQuery()` makes internally on your behalf. It fires synchronously with the current state before the transport is involved, and is unguarded, so the throw propagates back out of the call *you* made, unlogged. Out of `.subscribe()` that leaves your subscriber registered with no unsubscribe function returned. Out of `liveQuery()` it is worse: you get no handle at all, so there is nothing to `.close()`; the backfill never starts, so `initial()` is never called; and the stream opened a moment earlier keeps running (connected, reconnecting on its own) with nothing referencing it. Because the backfill never ran, the buffering phase also never ends: your `next()` is never called at all, and every event the stream receives is appended to a buffer you have no way to reach. Passing `opts.signal` is the only way to stop it. Since this call always happens, it is the *first* thing a throwing `status` handler does. +- **Delivery to other consumers stops at the one that threw**: later subscribers and any concurrent `for await`. For the iterator the event is *dropped*, not queued, because the fan-out runs before the event is handed to a waiter or buffered. And if the throw lands on the terminal `closed` status, the iterator is never marked done either, so a `for await` waits forever against a dead stream until something calls `.close()`. Order-dependent and invisible. +- **A throwing `status` handler subscribed before `.connected()`** makes that promise reject with its timeout error against a stream that is already `live`, because the throw aborts the fan-out before `connected()`'s internal watcher is reached. +- **Inside `liveQuery()`**, a throw from `initial()`, or from `next()` during the backfill flush, is absorbed by the backfill's own error path, with no log and no signal. The cost differs: `initial()` throws *before* the flush starts, so every buffered event is lost, while a `next()` throw mid-flush loses only the remainder. Either way, later live events reach `next()` as normal, with a further throw isolated and logged. + +`SSE_AUTH_ERROR` is the one caller-side failure that isn't terminal. A rejecting `auth` callback propagates out of a REST call, but on a stream, where `auth` is invoked on every connection attempt rather than once per stream, a token endpoint having a bad minute is treated as transient and retried, rather than tearing down a stream that is otherwise healthy. + + + + +The SDK never panics on API or network failures, mirroring the TypeScript SDK's "never throws" guarantee. Request-response operations (queries, ingest, pipes, admin) return `(T, error)`, while result-less ones (`Pipes.Set`/`Delete`, `Policy.Set`, `Schema.Refresh`, `Sys.Health`) return a bare `error`. HTTP exchange errors are `*wavehouse.Error` (unwrap via `errors.As`, or use `wavehouse.IsRetryable(err)` to shortcut the `errors.As` + `.Retryable` check); client-side failures such as an `Auth` provider or marshal error are plain wrapped errors, so handle the `errors.As == false` case; see the [worked example](/sdk/setup/go#error-handling). Streaming methods (`Stream`, `Subscribe`, `Close`) report through the subscriber's `Error` callback instead, and `Connected(ctx)` returns plain errors. + +| Status | Code | Retryable | Description | +|--------|------|-----------|--------------| +| 400 | `HTTP_400` | No | Bad request (validation, missing fields) | +| 401 | `HTTP_401` | No | Invalid or expired JWT (missing tokens use `default_role`, resulting in success or 403) | +| 403 | `HTTP_403` | No | Insufficient permissions | +| 404 | `HTTP_404` | No | Table or pipe not found | +| 429 | `HTTP_429` | Yes | Rate limited (auto-retries, honoring `Retry-After`, capped at 30s) | +| 500 | `HTTP_500` | Yes | Server error (retried per `ClientOptions.MaxRetries`) | +| 503 | `HTTP_503` | Yes | Service unavailable (auto-retries, honoring `Retry-After`, capped at 30s) | +| 0 | `NETWORK_ERROR` | Yes | Network failure (retried with exponential backoff) | +| 0 | `ABORTED` | No | Request canceled via `context.Context` | +| *3xx* | `REDIRECT` | No | The request carried a credential and was redirected; the SDK refused to follow it | +| 0 | `SSE_AUTH_ERROR` | Yes | The `Auth` provider returned an error for this attempt; the stream retries, so a token endpoint having a bad minute doesn't tear down a healthy stream | +| 0 | `SSE_NETWORK_ERROR` | Yes | Transport failure opening or holding the stream connection | +| 0 | `SSE_CONNECT_ERROR` | No | `BaseURL` is unparseable, or its scheme is not `http`/`https`; retrying cannot fix it | +| *3xx* | `SSE_REDIRECT` | No | The stream endpoint redirected while the request carried a credential, and the SDK refused to follow it | +| 200 | `SSE_BAD_CONTENT_TYPE` | No | A `200` that wasn't `text/event-stream`; something between you and WaveHouse answered (a captive portal, an auth gateway's login page) | +| 0 | `SSE_PARSE_ERROR` | Yes | A frame's JSON didn't decode; the frame is dropped and the stream continues | +| 0 | `SSE_READ_ERROR` | Yes | The connection failed mid-read; the stream reconnects from the last event ID | +| 0 | `SSE_ERROR` | Yes | Stream failure the SDK could not classify further | + +Retries apply to all HTTP methods, matching the TypeScript SDK's `http.ts`. For `/v1/ingest`, at-least-once delivery on retry is a documented contract (see ["At-least-once on retry"](/api#post-v1ingesttabletable--ingest-data)); use server-side dedup to suppress duplicates. `/v1/ops/query` (raw SQL) requires `admin_role`, so repeated execution on retry is an accepted risk. + + + + +--- + +## Full API Tree + + + + +```text +createClient(config) → WaveHouseClient +├── .from(table) → TableRef (NOT thenable) +│ ├── .fetch(opts?) → Promise> +│ ├── .select(...cols?) → QueryBuilder (PromiseLike) +│ │ ├── .select() .selectAll() .where() .count() .sum() .avg() .min() .max() +│ │ │ .countDistinct() .aggregate() .groupBy() .orderBy() +│ │ │ .limit() .timeRange() .cacheTTL() +│ │ ├── .fetch(opts?) → Promise> +│ │ ├── .stream(opts?) → StreamController +│ │ └── .liveQuery(subscriber, opts?) → LiveQuery +│ ├── .selectAll() → QueryBuilder (PromiseLike) +│ ├── .insert(data) → Promise> +│ ├── .insertNDJSON(source) → Promise> +│ ├── .schema() → Promise> (admin) +│ └── .stream(opts?) → StreamController +├── .pipe(name, params?) → PipeRef (PromiseLike) +│ ├── .fetch(opts?) → Promise> // { signal } only, no limit +│ └── .stream(opts?) → StreamController +├── .pipes (admin) +│ ├── .list() → Promise> +│ ├── .get(name) → Promise> +│ ├── .set(name, def) → Promise> +│ └── .delete(name) → Promise> +├── .sql(query, opts?) → Promise> (admin) +├── .schema (admin) +│ ├── .list() → Promise> +│ └── .refresh() → Promise> +├── .policy (admin) +│ ├── .get() → Promise> +│ ├── .set(policy) → Promise> +│ └── .validate(policy) → Promise> +├── .dlq (admin) +│ ├── .list() → Promise> +│ ├── .table(name) → Promise> +│ └── .stream() → StreamController // not yet functional server-side, #197 +└── .sys + └── .health() → Promise> + +StreamController (NOT thenable) +├── .subscribe({ next, status?, error? }) → unsubscribe() +├── .connected(timeoutMs?) → Promise +├── .close() +├── .status → StreamStatus +└── [Symbol.asyncIterator]() → AsyncIterableIterator +``` + + + + +```text +NewClient(Config) → *Client +├── .From(table) → *TableRef +│ ├── .Fetch(ctx) → (*Page[map[string]any], error) +│ ├── .Select(...cols) → *QueryBuilder +│ │ ├── .Select() .SelectAll() .Where() .Count() .Sum() .Avg() .Min() .Max() +│ │ │ .CountDistinct() .Aggregate() .GroupBy() .OrderBy() +│ │ │ .Limit() .TimeRange() .CacheTTL() +│ │ ├── FetchTyped[Row](ctx, q) → (*Page[Row], error) // package-level generic func +│ │ ├── .FetchUntyped(ctx) → (*Page[map[string]any], error) +│ │ ├── .Stream(opts) → *StreamController +│ │ └── .LiveQuery(sub, opts) → *LiveQueryHandle +│ ├── .SelectAll() → *QueryBuilder +│ ├── .Insert(ctx, data) → (*InsertResult, error) +│ ├── .InsertNDJSON(ctx, ndjson) → (*InsertResult, error) +│ ├── .Schema(ctx) → (*TableSchema, error) +│ └── .Stream(opts) → *StreamController +├── .Pipe(name, params) → *PipeRef +│ ├── Fetch[Row](ctx, p) → ([]Row, error) // package-level generic func +│ ├── .FetchUntyped(ctx) → ([]map[string]any, error) +│ └── .Stream(opts) → *StreamController +├── .Pipes (admin) → *PipesNamespace +│ ├── .List(ctx) → ([]Pipe, error) +│ ├── .Get(ctx, name) → (*Pipe, error) +│ ├── .Set(ctx, name, PipeDef) → error +│ └── .Delete(ctx, name) → error +├── SQL[Row](ctx, client, query) → ([]Row, error) // package-level generic func, admin-only +├── .Schema (admin) → *SchemaNamespace +│ ├── .List(ctx) → (Schemas, error) +│ └── .Refresh(ctx) → error +├── .Policy (admin) → *PolicyNamespace +│ ├── .Get(ctx) → (*Policy, error) +│ ├── .Set(ctx, *Policy) → error +│ └── .Validate(ctx, *Policy) → (*ValidationResult, error) +├── .DLQ (admin) → *DLQNamespace +│ ├── .List(ctx) → (*DLQStats, error) +│ ├── .Table(ctx, name) → (*DLQStats, error) +│ └── .Stream(opts) → *StreamController // not yet functional server-side, #197 +└── .Sys → *SysNamespace + └── .Health(ctx) → error + +*StreamController +├── .Subscribe(*StreamSubscriber) → func() // unsubscribe +├── .Events() → <-chan StreamEvent // idiomatic Go alternative to an async iterator +├── .Close() +├── .Status() → StreamStatus +└── .Connected(ctx) → error // Go-only addition, blocks until live +``` + + + + +--- + +## Row types + +Both SDKs decode rows into a type you supply. TypeScript generates that type from a running server; Go asks you to write the struct. + + + + +The package ships a `wavehouse-codegen` bin, so after installing `@wavehouse/sdk` you can run it with `npx`. It reads the admin-only `/v1/ops/schema` endpoint, so a non-dev server needs an admin token or returns `403`: + +```bash +npx wavehouse-codegen --url http://localhost:8080 --out ./src/db.d.ts + +# Or, working inside this repo (clients/ts/): +pnpm codegen --url http://localhost:8080 --out ./src/db.d.ts +``` + +Pass an admin-role token with `--auth `. + +**Options:** + +| Flag | Description | Default | +|------|-------------|---------| +| `--url`, `-u` | WaveHouse base URL | `http://localhost:8080` | +| `--out`, `-o` | Output .d.ts file path | `./wavehouse.d.ts` | +| `--auth`, `-a` | Bearer token (if auth required) | none | + +**Example output:** + +```ts +// Auto-generated by @wavehouse/sdk codegen +export interface Database { + clicks: ClicksRow; + events: EventsRow; +} + +export interface ClicksRow { + event_id: string; + page: string; + user_id: string; + duration_ms: number; + received_timestamp: string; +} +``` + +**ClickHouse → TypeScript type mapping:** + +| ClickHouse Type | TypeScript Type | +|----------------|-----------------| +| `String`, `FixedString`, `UUID`, `DateTime*`, `Date*`, `Enum*`, `IPv4/6` | `string` | +| `UInt*`, `Int*`, `Float*` | `number` | +| `Decimal*` | `number` *(generated)*, but `/v1/query` returns Decimals as **quoted strings**, so treat the field as `string` until codegen is fixed ([#453](https://github.com/Wave-RF/WaveHouse/issues/453)) | +| `Bool` | `boolean` | +| `Nullable(T)` | `T \| null` | +| `Array(T)` | `T[]`, except `Array(UInt8)`, which `/v1/query` base64-encodes, so the generated `number[]` is a `string` at runtime ([#436](https://github.com/Wave-RF/WaveHouse/issues/436)) | +| `Map(K, V)` | `Record` | +| `LowCardinality(T)` | same as `T` | + + + + +There is no Go codegen CLI. Write the row struct and tag each field with its column name; `FetchTyped[Row]`, `Fetch[Row]`, and `SQL[Row]` decode into whatever shape you define, and `map[string]any` with the untyped methods covers a dynamic or one-off schema. + +```go +package myapp + +// ClicksRow is one row of the "clicks" table. +type ClicksRow struct { + Page string `json:"page"` + Button string `json:"button"` + Score float64 `json:"score"` + ReceivedTimestamp *string `json:"received_timestamp,omitempty"` +} +``` + +Give a column with a `DEFAULT` a pointer field with `,omitempty`, as `received_timestamp` has above: `nil` omits the field so the server default applies, while a pointer to the zero value still sends an explicit `0`/`false`/`""`. + +**ClickHouse → Go field types:** + +| ClickHouse Type | Go Type | +|------------------|---------| +| `String`, `FixedString`, `UUID`, `DateTime*`, `Date*`, `Time`/`Time64`, `Enum8`/`Enum16`, `IPv4`/`IPv6` | `string` | +| `Bool` / `Boolean` | `bool` | +| `UInt8` / `UInt16` / `UInt32` / `UInt64` | `uint8` / `uint16` / `uint32` / `uint64` | +| `Int8` / `Int16` / `Int32` / `Int64` | `int8` / `int16` / `int32` / `int64` | +| `Float32`, `BFloat16` | `float32` | +| `Float64` | `float64` | +| `UInt128`/`UInt256`, `Int128`/`Int256` | `json.Number` | +| `Decimal*` | `string` | +| `Nullable(T)` | `*T` | +| `LowCardinality(T)` | same as `T` | +| `Array(T)` | `[]T`, except `Array(UInt8)` → `json.RawMessage` ([#436](https://github.com/Wave-RF/WaveHouse/issues/436)) | +| `Map(K, V)` | `map[K]V`, but only where `K` is a string or integer type: `encoding/json` cannot use `bool`, float, or pointer map keys, so reach for `map[string]any` there | +| `SimpleAggregateFunction(fn, T)` | same as `T` | + +Go keeps ClickHouse integer **widths** (`UInt64` → `uint64`, not a generic `number`), so 64-bit columns decode exactly where TS hits the 2^53 ceiling. These types target `/v1/query` and `/v1/pipes/*`; for the raw-SQL path (`/v1/ops/query`), which quotes 64-bit-and-wider integers, use `map[string]any` with `SQL[Row]`. + + + + +--- + +## Testing + +The cross-language wire-format **conformance suite** replays one shared fixture (`clients/go/testdata/wire_cases.json`) through both SDKs, asserting HTTP methods, paths, content types, and bodies. The Go half is `clients/go/conformance_test.go` (run by `make test-sdk-go`); the TypeScript half is `tests/conformance/conformance_ts.mjs` (run by `make test-sdk-ts`). `make test-sdk` runs both, and so does CI, keeping the wire formats in step. + + + + +The SDK doubles as the E2E integration test harness. Tests in `tests/e2e/sdk/` exercise the full pipeline (ingest → ClickHouse → query) through the SDK, validating both the backend and the client library in one pass. + +```bash +# Unit tests + wire conformance +make test-sdk-ts + +# Run all E2E tests: the orchestrator boots a ClickHouse testcontainer + +# the wavehouse-cov binary, then runs the SDK suite +make test-e2e +``` + +Test files live in `tests/e2e/sdk/` (each `*.test.ts`): `admin`, `auth`, `batching`, `cache`, `dlq`, `ingest`, `ndjson`, `query`, `streaming`, `stress`, plus `helpers`, a stack-free unit test of the harness's own `waitForCondition` poll helper rather than a pipeline test. + +See [Development Guide: E2E Tests via SDK](/development#e2e-tests-via-sdk) for architecture details and workflow tips. + + + + +Unit tests are colocated in `clients/go/`, which is its own module (`clients/go/go.mod`), separate from the root `WaveHouse` module: + +```bash +# Unit tests + wire conformance + coverage gate +make test-sdk-go +``` + +E2E tests (build tag `e2e`) run against a live WaveHouse instance: + +```bash +WAVEHOUSE_URL=http://localhost:8080 WAVEHOUSE_AUTH='' make test-sdk-go-e2e +``` + +`WAVEHOUSE_URL` defaults to `http://localhost:8080`, and the optional `WAVEHOUSE_AUTH` covers the admin cases; the suite skips if the server is unreachable. Unlike the TypeScript SDK, the Go E2E suite isn't yet driven by the repo's `make test-e2e` orchestrator. Wiring it in, with its own coverage gate, is tracked in [#518](https://github.com/Wave-RF/WaveHouse/issues/518). + + + diff --git a/docs/src/content/docs/sdk/setup/go.md b/docs/src/content/docs/sdk/setup/go.md new file mode 100644 index 00000000..23332923 --- /dev/null +++ b/docs/src/content/docs/sdk/setup/go.md @@ -0,0 +1,171 @@ +--- +title: "Go SDK setup" +description: "Installing the WaveHouse Go client, creating a client, typed rows via generics, and the (T, error) model." +--- + +Setup and caveats for `github.com/Wave-RF/WaveHouse/clients/go`, the Go client: installation, client construction, typed rows, and the `(T, error)` model. Usage is documented per topic, both languages side by side, starting at [Queries](/sdk/queries). Writing TypeScript instead? [TypeScript setup](/sdk/setup/typescript) is the mirror of this page. + +## Installation + +```bash +go get github.com/Wave-RF/WaveHouse/clients/go +``` + +Requires Go 1.24+ (the `go.mod` floor, which tracks supported releases rather than the server's patch-pinned toolchain). + +```go +import wavehouse "github.com/Wave-RF/WaveHouse/clients/go" +``` + +The `wavehouse` alias is optional but keeps call sites short; all examples here assume it. + +## Quick Start + +```go +package main + +import ( + "context" + "fmt" + "log" + + wavehouse "github.com/Wave-RF/WaveHouse/clients/go" +) + +func main() { + wh := wavehouse.NewClient(wavehouse.Config{ + BaseURL: "http://localhost:8080", + Auth: wavehouse.StaticToken("your-jwt"), + }) + + page, err := wh.From("clicks"). + Select("page", "button"). + Where("page", wavehouse.OpEq, "/home"). + Limit(10). + FetchUntyped(context.Background()) + if err != nil { + log.Fatal(err) + } + for _, row := range page.Data { + fmt.Println(row["page"], row["button"]) + } +} +``` + +## Creating a Client + +`Auth` is any function returning a token; `wavehouse.StaticToken(token)` wraps a fixed one, as in the Quick Start above. + +```go +wh := wavehouse.NewClient(wavehouse.Config{ + BaseURL: "https://wavehouse.example.com", + Auth: func(ctx context.Context) (string, error) { + return myAuthProvider.GetToken(ctx) + }, +}) +``` + +### `Config` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `BaseURL` | `string` | none | Required WaveHouse server URL, optionally with a path prefix. A trailing `/` is trimmed and every request path is appended on both transports, so a server under `https://app.example.com/wavehouse` works as-is. | +| `Auth` | `func(context.Context) (string, error)` | `nil` | Token provider called before each request. `nil` means unauthenticated access. | +| `Options` | `*ClientOptions` | `nil` | Transport tuning (see below). | +| `HTTPClient` | `*http.Client` | fresh `&http.Client{}` | Override for custom TLS, proxies, or test transports. | + +:::caution[Timeouts: use contexts, not `http.Client.Timeout`] +The default client sets no `Timeout`; use a `context.Context` deadline to prevent hangs. Leave `Timeout` unset on a custom `HTTPClient` too: it would kill long-lived SSE streams and force reconnect loops. Use `Transport`-level dial/TLS/response-header timeouts instead. +::: + +### `ClientOptions` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `MaxRetries` | `*int` | `2` | Retry attempts for retryable errors (5xx, 429, network failures). `nil` takes the default; `wavehouse.Ptr(0)` turns retries off. A negative value is treated as `0`. | +| `Headers` | `map[string]string` | `nil` | Sent on every request the client makes: REST calls and SSE streams alike. | + +`*Client` is safe for concurrent use (state is immutable after `NewClient` and builder chains copy), provided your `Auth` function is concurrency-safe. + +`Headers` is the Go analog of the TypeScript SDK's [`options.headers`](/sdk/setup/typescript#custom-headers): a gateway credential, a tenant selector, or tracing metadata with no first-class option. It is also how an operator sends the server's non-JWT [operator key](/api#authentication): + +```go +wh := wavehouse.NewClient(wavehouse.Config{ + BaseURL: "http://localhost:8080", + Options: &wavehouse.ClientOptions{ + Headers: map[string]string{"X-Operator-Key": os.Getenv("WH_OPERATOR_KEY")}, + }, +}) +``` + +The SDK's own headers win: `Authorization`, `Accept`, `Content-Type`, and the stream's `Cache-Control` are set after yours and overwrite any collision, matched case-insensitively and replacing rather than appending. The map is copied at `NewClient`, so later mutation changes nothing. Because a configured header can carry a credential, the SDK refuses to follow a redirect on any request that carries one, on REST calls and streams alike: `net/http` strips only its own four sensitive headers across hosts and forwards yours verbatim. A refused REST redirect surfaces as a non-retryable `REDIRECT`; a stream reports `SSE_REDIRECT`. There is no Go field for `options.fetch` or `options.fetchOptions` because `Config.HTTPClient` covers both: supply your own `*http.Client`, or a custom `http.RoundTripper` on its `Transport`. + +:::note[How the token is transmitted] +The SDK sends `Authorization: Bearer ` on every request, SSE streams included, and never uses a `?token=` query fallback. The TypeScript SDK streams over `fetch` rather than `EventSource` for exactly this reason, so header auth is shared behavior rather than a Go-only property (see its [equivalent note](/sdk/setup/typescript#creating-a-client)). The token is re-read from `Auth` on every reconnect attempt, so a rotating token keeps a long-lived stream alive. +::: + +:::caution[A credentialed stream will not follow a redirect] +When the stream request carries a credential (an `Auth` token or a `ClientOptions.Headers` entry), the SDK refuses any 3xx and fails the stream with a terminal `SSE_REDIRECT`. Following it would either strip `Authorization` on a cross-host hop and silently downgrade the stream to `default_role`, or forward your configured headers to wherever the redirect points. Uncredentialed streams follow redirects normally. +::: + +Use `https://` for any authenticated server outside a trusted network. The SDK allows `http://` for local development and private networks, but bearer tokens over plaintext are insecure. + +## Typed Rows (Generics) + +Pass a row type parameter to decode results into your struct instead of `map[string]any`: + +```go +type ClickRow struct { + Page string `json:"page"` + Button string `json:"button"` + DurationMS int `json:"duration_ms"` +} + +page, err := wavehouse.FetchTyped[ClickRow](ctx, + wh.From("clicks").Select("page", "button", "duration_ms").Limit(100), +) +// page.Data is []ClickRow +``` + +Row structs are hand-written; see [Row types](/sdk/reference#row-types) for the ClickHouse-to-Go field mapping. `FetchTyped`, `Fetch[Row]` (pipes), and `SQL[Row]` (raw SQL) are package-level generic functions because Go lacks generic methods; the untyped equivalents (`.FetchUntyped(ctx)`) are ordinary methods. + +## Error Handling + +Request-response operations (queries, ingest, pipes, admin) return `(T, error)`, or a bare `error` when there is no body (`Pipes.Set`/`Delete`, `Policy.Set`, `Schema.Refresh`, `Sys.Health`). HTTP exchange errors are `*wavehouse.Error`; unwrap via `errors.As`. Client-side failures (`Auth` provider, marshal errors) are plain wrapped errors, so handle the `errors.As == false` case too. Streaming methods (`Stream`, `Subscribe`, `Close`, `Connected`) report through callbacks or plain errors; see [Streaming](/sdk/streaming). + +```go +page, err := wh.From("clicks").Fetch(ctx) +if err != nil { + var whErr *wavehouse.Error + if errors.As(err, &whErr) { + fmt.Println(whErr.Status, whErr.Code, whErr.Message, whErr.Retryable) + } else { + fmt.Println("client-side failure:", err) // auth provider, marshal, ... + } + return err +} +``` + +See [Reference → Error Handling](/sdk/reference#error-handling) for retry behavior and error codes. + +## Differences from the TypeScript SDK + +Both SDKs share a wire format and feature set, verified in CI against a shared `wire_cases.json` fixture that asserts equivalent HTTP requests for builder calls. The API shapes differ: + +- **No `Result` union.** Go returns `(T, error)`; a non-nil `error` is the only failure signal. No `{ok, data, error}` objects, no `error: null` sentinels. +- **`context.Context` instead of `AbortSignal`.** Non-streaming calls take `ctx context.Context` first; use a deadline or `cancel()`. See [Reference → Cancellation](/sdk/reference#cancellation). +- **Streams closed explicitly.** `TableRef.Stream` and `QueryBuilder.Stream` take no context; the returned `*StreamController` owns its goroutine and connection until `.Close()` (usually deferred). See [Streaming](/sdk/streaming). +- **Generics on package functions.** Go has no type parameters on methods, so use `FetchTyped[Row]`, `Fetch[Row]`, or `SQL[Row]`. +- **No implicit "await."** `QueryBuilder` is not `PromiseLike`; call `.FetchUntyped(ctx)` or `wavehouse.FetchTyped[Row](ctx, builder)` explicitly. +- **No third-party dependencies.** Stdlib only, SSE frame parser included. The TypeScript SDK carries exactly one runtime dependency (`eventsource-parser`, ~1.4 KB gzipped). +- **Any slice batches.** Reflection lets `[]ClickRow{...}` take the same NDJSON batch path as `[]map[string]any`. See [Queries → Insert](/sdk/queries#inserting-rows). + +## Where to go next + +The topic pages cover both SDKs, tabbed by language. The tab you pick here follows you across all of them. + +- [Queries](/sdk/queries): Tables, chainable query builder, inserts, pagination, and raw SQL. +- [Streaming & Live Queries](/sdk/streaming): SSE streams, client-side filtering, and backfill-then-live queries. +- [Pipes](/sdk/pipes): Execute and manage named query pipes. +- [Admin & System](/sdk/admin): Schema introspection, access-control policy, DLQ stats, and health checks. +- [Reference & CLI](/sdk/reference): Error codes, cancellation, the full API tree, and row types. diff --git a/docs/src/content/docs/sdk/setup/index.md b/docs/src/content/docs/sdk/setup/index.md new file mode 100644 index 00000000..cd1f47be --- /dev/null +++ b/docs/src/content/docs/sdk/setup/index.md @@ -0,0 +1,13 @@ +--- +title: "SDK setup" +description: "Package names, install commands, and version floors for the WaveHouse TypeScript and Go clients." +--- + +Setup is the only per-language part. Everything after it is documented once per topic, both languages side by side, starting at [Queries](/sdk/queries). + +| SDK | Package | Install | Requires | +| --- | ------- | ------- | -------- | +| [TypeScript](/sdk/setup/typescript) | `@wavehouse/sdk` | `npm install @wavehouse/sdk` | Node 22+, or any modern browser | +| [Go](/sdk/setup/go) | `github.com/Wave-RF/WaveHouse/clients/go` | `go get github.com/Wave-RF/WaveHouse/clients/go` | Go 1.24+ | + +Both have full API-tree parity, so pick the language your service already uses. diff --git a/docs/src/content/docs/sdk/setup/typescript.mdx b/docs/src/content/docs/sdk/setup/typescript.mdx new file mode 100644 index 00000000..31245dd3 --- /dev/null +++ b/docs/src/content/docs/sdk/setup/typescript.mdx @@ -0,0 +1,567 @@ +--- +title: "TypeScript SDK setup" +description: "Installing @wavehouse/sdk, creating a client, custom headers and fetch, and the Result type." +--- + +import { Tabs, TabItem, LinkCard, CardGrid } from "@astrojs/starlight/components"; + +Setup and caveats for `@wavehouse/sdk`, the TypeScript client: installation, runtimes, client construction, and the `Result` error model. Usage is documented per topic, both languages side by side, starting at [Queries](/sdk/queries) — writing Go instead? [Go setup](/sdk/setup/go) is the mirror of this page. + +`@wavehouse/sdk` has exactly one runtime dependency — `eventsource-parser` (~1.4 KB gzipped, itself dependency-free), which frames the SSE stream. + +## Installation + + + + +```bash +pnpm add @wavehouse/sdk +``` + + + + +```bash +npm install @wavehouse/sdk +``` + + + + +```bash +yarn add @wavehouse/sdk +``` + + + + +```bash +bun add @wavehouse/sdk +``` + + + + +```bash +deno add npm:@wavehouse/sdk +``` + + + + No bundler or package manager required — the ES module loads natively in modern browsers: + +```html + +``` + +Pin a version for production (`https://esm.sh/@wavehouse/sdk@0.1.0`); jsDelivr (`.../+esm`) and unpkg (`?module`) serve the same module. For pages that can't use ES modules, the bundled IIFE build at `https://cdn.jsdelivr.net/npm/@wavehouse/sdk` exposes a `WaveHouse` global (`WaveHouse.createClient({ … })`) for a classic ` + +``` + +### Runtime support + +**Browsers** — all SDK features work natively. `fetch`, `AbortController`, and `ReadableStream` are built into every modern browser; no polyfills are required. + +**Node.js** — every feature, streaming included, works in Node 22 and later (the package's minimum, per `engines.node`). 22 is the only line we test against, and older releases are past end-of-life upstream. Streaming runs on the same `fetch` and `ReadableStream` globals as the rest of the SDK, so it needs **no `EventSource` polyfill** — earlier versions did, and that requirement is gone. + +## Quick Start + + + + +```ts +import { createClient } from '@wavehouse/sdk'; + +const wh = createClient({ + baseURL: 'http://localhost:8080', + auth: async () => getAccessToken(), // omit for public/unauthenticated +}); + +// Query +const { data, error } = await wh.from('clicks').select('page').limit(10); + +// Insert +await wh.from('clicks').insert({ page: '/home', button: 'signup' }); + +// Stream +const stream = wh.from('clicks').stream(); +const unsub = stream.subscribe({ + next: (event) => console.log(event.data), + status: (s) => console.log('Stream:', s), +}); +``` + + + + +```ts +import { createClient } from '@wavehouse/sdk'; + +const wh = createClient({ + baseURL: 'http://localhost:8080', + auth: async () => getAccessToken(), // omit for public/unauthenticated +}); + +// Query +const { data, error } = await wh.from('clicks').select('page').limit(10); + +// Insert +await wh.from('clicks').insert({ page: '/home', button: 'signup' }); + +// Stream +const stream = wh.from('clicks').stream(); +const unsub = stream.subscribe({ + next: (event) => console.log(event.data), + status: (s) => console.log('Stream:', s), +}); +``` + + + + +```ts +import { createClient } from '@wavehouse/sdk'; + +const wh = createClient({ + baseURL: 'http://localhost:8080', + auth: async () => getAccessToken(), // omit for public/unauthenticated +}); + +// Query +const { data, error } = await wh.from('clicks').select('page').limit(10); + +// Insert +await wh.from('clicks').insert({ page: '/home', button: 'signup' }); + +// Stream +const stream = wh.from('clicks').stream(); +const unsub = stream.subscribe({ + next: (event) => console.log(event.data), + status: (s) => console.log('Stream:', s), +}); +``` + + + + +```ts +import { createClient } from '@wavehouse/sdk'; + +const wh = createClient({ + baseURL: 'http://localhost:8080', + auth: async () => getAccessToken(), // omit for public/unauthenticated +}); + +// Query +const { data, error } = await wh.from('clicks').select('page').limit(10); + +// Insert +await wh.from('clicks').insert({ page: '/home', button: 'signup' }); + +// Stream +const stream = wh.from('clicks').stream(); +const unsub = stream.subscribe({ + next: (event) => console.log(event.data), + status: (s) => console.log('Stream:', s), +}); +``` + + + + +```ts +import { createClient } from '@wavehouse/sdk'; + +const wh = createClient({ + baseURL: 'http://localhost:8080', + auth: async () => getAccessToken(), // omit for public/unauthenticated +}); + +// Query +const { data, error } = await wh.from('clicks').select('page').limit(10); + +// Insert +await wh.from('clicks').insert({ page: '/home', button: 'signup' }); + +// Stream +const stream = wh.from('clicks').stream(); +const unsub = stream.subscribe({ + next: (event) => console.log(event.data), + status: (s) => console.log('Stream:', s), +}); +``` + + + + +```ts +import { createClient } from 'https://esm.sh/@wavehouse/sdk'; + +const wh = createClient({ + baseURL: 'http://localhost:8080', + auth: async () => getAccessToken(), // omit for public/unauthenticated +}); + +// Query +const { data, error } = await wh.from('clicks').select('page').limit(10); + +// Insert +await wh.from('clicks').insert({ page: '/home', button: 'signup' }); + +// Stream +const stream = wh.from('clicks').stream(); +const unsub = stream.subscribe({ + next: (event) => console.log(event.data), + status: (s) => console.log('Stream:', s), +}); +``` + + + + +## Creating a Client + +```ts +import { createClient } from '@wavehouse/sdk'; +import type { Database } from './my-types'; // optional hand-written types + +const wh = createClient({ + baseURL: 'https://.wavehouse.app', + auth: async () => myAuthProvider.getToken(), + options: { + maxRetries: 2, + }, +}); +``` + +### `ClientConfig` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `baseURL` | `string` | — | WaveHouse server URL, optionally including a path prefix (required) | +| `auth` | `() => Promise \| string` | — | Token provider. Omit for public access | +| `options.maxRetries` | `number` | `2` | Retry attempts for failed/5xx **REST** requests; stream reconnects are unbounded ([details](/sdk/streaming#transport-behavior)) | +| `options.headers` | `Record` | — | Headers added to every request ([details](#custom-headers)) | +| `options.fetchOptions` | `RequestInit` | — | Extra `RequestInit` fields merged into every request ([details](#extra-requestinit-fields)) | +| `options.fetch` | `FetchLike` | global `fetch` | HTTP implementation for every request ([details](#supplying-your-own-fetch)) | + +:::note[How the token is transmitted] +The SDK attaches your `auth` token as an `Authorization: Bearer` header on every request, streaming included. It is re-read on each connection attempt, so a stream that outlives its token reconnects with a fresh one instead of replaying an expired credential. + +The token is never placed in the URL. The server still accepts a `?token=` query parameter for clients that cannot set headers — see [API Reference — Authentication](/api#authentication) — but the SDK does not use it, so it never reaches a proxy or CDN access log. +::: + +:::caution[Streaming asks more of a custom `fetch`] +`.stream()` and `.liveQuery()` read the response as it arrives, so an `options.fetch` used with them must return a response with a live streaming body. A response handed back with an absent or already-consumed body fails fast with `SSE_NO_STREAM_BODY`. What the SDK cannot rescue is a wrapper that *awaits* the body before returning — `await res.text()`, or the `res.clone().text()` a logging wrapper reaches for — because on a stream that never ends it never resolves and your function never returns. Both satisfy every REST call, which is what makes the trap easy to walk into. + +`options.headers` is also subject to CORS in a browser: a header outside the safelist joins the preflight, which the origin has to allow. WaveHouse advertises a fixed set, so custom headers reach it cross-origin only when a proxy in front terminates the preflight — the deployment they exist for. Server-side callers never preflight. +::: + +#### Serving under a path prefix + +`baseURL` may carry a path prefix, for a WaveHouse reachable somewhere other than the root of an origin — behind a backend-for-frontend (BFF), an app-server route, or a path-routed ingress. Every request path is appended to it, on both transports: + +```ts +const wh = createClient({ baseURL: 'https://app.example.com/api/wavehouse' }); +// queries → https://app.example.com/api/wavehouse/v1/query +// streams → https://app.example.com/api/wavehouse/v1/stream +``` + +Trailing slashes on `baseURL` are optional, but it must be **absolute** — scheme and host included — and the scheme must be `http` or `https`. A same-origin relative path like `/api/wavehouse` makes every REST call reject with a `TypeError` instead of returning a `Result`, and surfaces on a stream as an `SSE_CONNECT_ERROR` to the subscriber's `error` callback — so build an absolute one: `` `${location.origin}/api/wavehouse` ``. A `ws://`/`wss://` base ends the stream with the same error: streaming is plain HTTP, and there is no WebSocket endpoint. + +The proxy in front must **strip** the prefix before forwarding, since WaveHouse itself always serves at `/v1/…` — see [Behind a reverse proxy → Path prefixes](/reverse-proxy#path-prefixes). + +### Custom headers + +`options.headers` adds headers to every request, streaming included — the usual reason being a header-gated proxy in front of WaveHouse, such as a Cloudflare Access service token: + +```ts +const wh = createClient({ + baseURL: 'https://.wavehouse.app', + options: { + headers: { + 'CF-Access-Client-Id': process.env.CF_ACCESS_CLIENT_ID!, + 'CF-Access-Client-Secret': process.env.CF_ACCESS_CLIENT_SECRET!, + }, + }, +}); +``` + +Header names are matched **case-insensitively**, as HTTP requires, so `authorization` and `Authorization` are the same header rather than two. + +Your headers are applied *underneath* the SDK's own, and a collision means yours is dropped rather than merged: + +- **`auth` keeps `Authorization`.** Setting it here won't displace your token provider. Use `auth` for credentials. +- **`Content-Type` and `Accept` belong to the request.** The SDK knows what it's sending; a global `Content-Type` that outranked it would break requests whose body isn't JSON. + +Nothing is ever comma-joined: on a collision the SDK's value stands alone, and two of your own entries differing only in case collapse to the last one — a header joined rather than replaced is how you end up sending `Content-Type: application/json, image/png`. + +From a **browser**, a cross-origin custom header must also survive CORS preflight, and WaveHouse allow-lists a fixed set (`Accept`, `Authorization`, `Content-Type`, `Last-Event-ID`, `X-Request-ID`) with no config knob. So custom headers work server-side, or from a browser when the proxy in front owns CORS — which is the same proxy the header is usually for. + +Headers are static. For a credential that rotates per request, wrap the transport with [`options.fetch`](#supplying-your-own-fetch); a callback form is tracked in [#459](https://github.com/Wave-RF/WaveHouse/issues/459). + +### Extra `RequestInit` fields + +`options.fetchOptions` is merged into the `RequestInit` of every request, streaming included — for settings that aren't headers and don't warrant replacing the whole transport: + +```ts +const wh = createClient({ + baseURL: 'https://wavehouse.example.com', + options: { fetchOptions: { cache: 'no-store', mode: 'cors' } }, +}); +``` + +It carries any `RequestInit` field — `mode`, `cache`, `keepalive`, `credentials`, `redirect` — plus runtime-specific extensions such as Next.js's `next: { tags }`, which tags the cached response for on-demand revalidation. Only those runtime extensions need a cast; the standard fields are declared on `RequestInit` already. + +:::caution[Two of those fields don't apply to streams] +On `.stream()` and `.liveQuery()`'s live connection the transport keeps `cache` and `redirect` for itself, because there they're correctness rather than preference. It constrains one more without owning it: `credentials` stays yours, forwarded in a browser and dropped elsewhere because some runtimes throw if it is set at all. See [Supplying your own fetch](#supplying-your-own-fetch) for what each is set to and why. + +Everything else reaches streams normally — including `mode` from the example above. Only `cache` is different there: on a stream it is already `no-store`. +::: + +:::note[`credentials: 'include'` needs a proxy that owns CORS] +Sending cookies cross-origin is a common reason to reach for this, but it can't work against WaveHouse's own CORS: it deliberately never emits `Access-Control-Allow-Credentials` (it's a Bearer-token API), and the default `cors_allowed_origins: "*"` makes `include` a hard browser failure regardless. It applies where a fronting proxy answers CORS itself — see [Behind a reverse proxy](/reverse-proxy#header-and-auth-forwarding). Same-origin deployments already send cookies without it. +::: + +The fields the SDK controls — `method`, `headers`, `body`, and `signal` — always win, so this can't corrupt the request itself. In particular `headers` here is ignored; use `options.headers`, which merges properly. + +### Supplying your own `fetch` + +`options.fetch` replaces the HTTP implementation the SDK uses. Reach for it to route through a proxy, attach client certificates, wrap requests in your own middleware (logging, tracing, circuit breaking), stub HTTP in your own tests without monkey-patching a global, or to work around transport behavior of the runtime you happen to be on: + +```ts +const wh = createClient({ + baseURL: 'https://.wavehouse.app', + options: { + fetch: async (url, init) => { + const started = performance.now(); + const res = await fetch(url, init); + console.log(res.status, url, `${Math.round(performance.now() - started)}ms`); + return res; + }, + }, +}); +``` + +Retries go through the same function, so middleware sees every attempt. + +The SDK always calls your function with a **string** URL and a plain `RequestInit`. Off the response the **REST path** reads `.ok` and `.headers` always, `.text()` on success, and `.status`, `.statusText` plus `.json()` when the response is not `ok` — so a hand-rolled response object needs all six there. A stub used with `.stream()` or `.liveQuery()` needs a different set: `.ok`, `.status`, `.type` (checked for `opaqueredirect`), `.headers` carrying `content-type: text/event-stream`, and a live, unread `.body` — plus `.statusText` and `.json()` if your stub ever answers a stream with a non-`ok` response, which goes through the same error parsing as REST. `.text()` is never called on a stream. + +**On REST**, if your function rejects, the request becomes a `NETWORK_ERROR` result and is retried with backoff — unless the `AbortSignal` you passed has been aborted, in which case the result is `ABORTED` and nothing is retried. That's decided from the signal rather than from what your function threw, so an implementation that signals abort some other way — `AbortSignal.timeout()` raises a `TimeoutError`, `node-fetch` its own `AbortError` class — is reported the same way. On a stream the same rejection is reported to the subscriber as `SSE_NETWORK_ERROR` and re-dialed — including an `AbortError`, since the stream transport applies the same rule: only an abort *it* raised, via `.close()`, ends the stream. See [Error Handling](/sdk/reference#error-handling). + +The option's type is exported as `FetchLike`. It is written out rather than as `typeof fetch`, which resolves differently depending on whether your TypeScript `lib` includes DOM. Its URL parameter is `string` — all the SDK ever passes — which, because function parameters are contravariant, accepts *more* implementations than the wider spelling would: the global `fetch` fits, and so does hand-written `(url: string, init?: RequestInit) => Promise` middleware. Let TypeScript infer your parameters (`(url, init) => …`) and it fits without annotation. + +Streaming goes through your function too. That comes with the extra requirement described above — the response must carry a live streaming body — and the streaming transport keeps ownership of two `RequestInit` fields where the value is load-bearing rather than a preference: `cache` (as an init field; a `Cache-Control` header would fail cross-origin preflight) and `redirect`. It constrains one more without owning it: `credentials` stays yours, forwarded in a browser and dropped elsewhere, because some runtimes throw if it is set at all. + +`redirect` depends on whether the request carries a credential. With one — an `auth` token or configured `headers` — a redirect is refused and reported as `SSE_REDIRECT`, because a cross-origin hop strips `Authorization` (which this endpoint answers with a silently reduced view rather than an error) while forwarding your other headers to wherever the redirect points. Without either, redirects are followed normally, so CDN canonicalization, geo/LB indirection, and an http→https upgrade all work. The test is that concrete pair rather than "is this request authenticated", and **cookies are not part of it**. A cookie isn't stripped the way `Authorization` is — it is re-derived from the cookie store at each hop, so a redirect carries it only while the hop stays inside **both** the request's origin and the cookie's own `Path`. Leaving either loses it. The origin half is the one no cookie attribute can override: under the default `same-origin` credentials mode the browser attaches nothing once a redirect goes cross-origin, whatever the `Domain` — so `app.example.com` → `api.example.com` on a shared `Domain=example.com` session cookie loses it, and so does an http→https upgrade, which is cross-origin by scheme and port. A same-origin rewrite from `/v1/stream` to `/stream` under a `Path=/v1` cookie loses it the other way. This endpoint answers a cookieless request with a reduced view rather than an error ([#478](https://github.com/Wave-RF/WaveHouse/issues/478)). Going cross-origin with `credentials: "include"` usually fails earlier and louder instead: WaveHouse never sends `Access-Control-Allow-Credentials`, so against it the browser blocks the response outright. Behind a proxy that owns CORS and does allow credentials the hop succeeds — and then ordinary cookie scoping decides. `include` *replaces* the origin rule rather than lifting it: the cookie crosses only if its own `Domain` covers the target, so a host-only cookie — what you get with no `Domain` attribute — still stops at its host. If you need to follow a credentialed redirect anyway, supply an `options.fetch` that overrides `redirect`. A `200` that isn't `text/event-stream` is refused too (`SSE_BAD_CONTENT_TYPE`), so an auth gateway's login page fails loudly instead of leaving the stream connected and permanently silent. + +#### Swapping in undici + +The motivating case is a transport bug in the runtime's own HTTP stack, which you can't fix from inside the SDK — for example [undici #5600](https://github.com/nodejs/undici/issues/5600): reusing a keep-alive socket while the event loop is idle stalls the request before it goes out. How bad it gets varies with the runtime and the idle gap: the upstream report measured ~450–465 ms against a 10 ms server, and we have measured anything from ~100 ms to tens of seconds on a server answering instantly. It affects undici 8.8.0–8.9.0, and Node 26 bundles 8.9.0. + +**Upgrading undici is the actual fix** — it landed in 8.10.0. What `options.fetch` buys you is a way to get there without waiting for a new runtime: install undici yourself and route requests through it. The one non-obvious part is that you must pass its dispatcher **explicitly**. + +```ts +import { createClient } from '@wavehouse/sdk'; +import { Agent, fetch as undiciFetch } from 'undici'; // npm install undici — 8.10.0+ + +// Explicit, not implied — see below. +const dispatcher = new Agent(); + +const wh = createClient({ + baseURL: 'https://.wavehouse.app', + options: { + fetch: (url, init) => + undiciFetch(url, { ...init, dispatcher } as never) as unknown as Promise, + }, +}); +``` + +:::caution[Importing a fixed undici is not enough on its own] +undici keeps its connection pool on a shared `globalThis` symbol, and whichever copy loads first claims it. Node claims it for the *bundled* copy the first time anything touches one of its web globals — a `fetch()` call, but equally a `new Headers()` or `new Response()` — not at startup. So which undici owns the pool comes down to a load order you don't really control, and auditing your own code for `fetch` calls won't tell you: any dependency can claim it first, and so can this SDK, which constructs a `Headers` on its abort and retry-exhausted paths. Call `undiciFetch` without a `dispatcher` and it resolves whatever is on that symbol, so requests can go through the buggy pool even though you imported a fixed undici. + +Measured with 8.9.0 loaded first and 8.10.0's `fetch` doing the request, 1.5 s idle gaps against a 10 ms server: + +```text + per-request ms +no explicit dispatcher 21 1514 1495 583 ← still stalling +explicit new Agent() 17 14 12 13 +``` + +An explicit `dispatcher` wins because the request never consults the shared symbol at all. An explicit `setGlobalDispatcher` call (below) wins the other way round — it overwrites the symbol after the first copy claimed it. +::: + +Both casts are load-bearing, for one underlying reason: undici declares its own request/response types, separate from the ones behind your global `fetch`, so the two aren't structurally assignable. `{ ...init, dispatcher } as never` covers the split on `RequestInit`, which differs on `body` and `headers` (`never` is assignable to either spelling, so one snippet compiles whether or not your `lib` includes DOM); and the return cast handles the `Response` mismatch. The URL needs no cast — `FetchLike` declares it as `string`, which undici's `RequestInfo` accepts. They're safe because of the narrow runtime contract above — a string URL and a plain `RequestInit` in, and only the response members enumerated above read back. `undici` is a dependency you add on top of `eventsource-parser`, the SDK's single ~1.4 KB runtime dependency. + +If you're genuinely pinned to an affected undici, the same snippet fixes it — your pinned version, with a dispatcher that never reuses a keep-alive socket: `new Agent({ pipelining: 0 })`. That costs a fresh connection per request, so prefer upgrading. Note that tuning `keepAliveTimeout` does **not** help: the retirement timer is starved by the same idle event loop that triggers the bug, so the socket is still there to be reused. + +To change the transport process-wide instead of per-client, an installed undici's `setGlobalDispatcher(new Agent())` is picked up by Node's built-in global `fetch`, no `options.fetch` needed. That's an explicit write, so it wins the race described above — but only if the *installed* undici is 8.10.0+, since it hands connection handling to that copy. + +### Type-Safe Tables + +Pass a `Database` type to get autocomplete on table names and row types: + +```ts +interface Database { + clicks: { page: string; button: string; score: number; received_timestamp: string }; + users: { id: string; name: string; email: string }; +} + +const wh = createClient({ baseURL: '...' }); +const clicks = wh.from('clicks'); // ✅ autocomplete +const { data } = await clicks.select('page', 'button').limit(10); +// data is Array<{ page: string; button: string; score: number; received_timestamp: string }> | null +``` + +Generate the `Database` interface from a running server with the [codegen CLI](/sdk/reference#row-types). + +## Result Type + +Every async SDK operation returns `Result` — a discriminated union that never throws for anything the server returns (caller and environment errors do throw — see [Error Handling](/sdk/reference#error-handling)): + +```ts +type Result = + | { ok: true; data: T; error: null; hasMore?: boolean; next?: () => Promise> } + | { ok: false; data: null; error: WaveHouseError } + +interface WaveHouseError { + status: number; // HTTP status (0 for network errors) + code: string; // e.g. 'HTTP_400', 'NETWORK_ERROR', 'ABORTED' + message: string; // Human-readable error message + details?: unknown; // Raw response body + retryable: boolean; // Whether SDK would retry this error +} +``` + +Usage pattern — branch on `result.ok` (or destructure `{ data, error }` if you prefer): + +```ts +const result = await wh.from('clicks').select('page').limit(10); +if (result.ok) { + console.log(result.data); // Row[] — TypeScript knows data is non-null here +} else { + console.error(result.error.message); // never throws +} +``` + +The full error-code table lives in [Error Handling](/sdk/reference#error-handling). + +## Where to go next + +The topic pages cover both SDKs, tabbed by language — the tab you pick here follows you across all of them. + + + + + + + + + diff --git a/docs/src/content/docs/sdk/streaming.md b/docs/src/content/docs/sdk/streaming.md deleted file mode 100644 index a7f8e3ed..00000000 --- a/docs/src/content/docs/sdk/streaming.md +++ /dev/null @@ -1,216 +0,0 @@ ---- -title: "SDK Streaming & Live Queries" -description: "Real-time SSE streams, client-side filtering, and backfill-then-live queries in @wavehouse/sdk." ---- - -Real-time consumption with `@wavehouse/sdk`: SSE event streams from tables, builders, and pipes, plus live queries that backfill history before going live. Builders and table refs come from [Queries](/sdk/queries). Examples import from `@wavehouse/sdk` or `https://esm.sh/@wavehouse/sdk` (see [Imports & Runtimes](/sdk#imports--runtimes)). - -## Streaming - -Streams are Server-Sent Events over `fetch`. The `auth` token rides in an `Authorization: Bearer` header — the same as every other request, on browsers and servers alike — and is re-read on each connection attempt, so a stream that outlives its token picks up a fresh one. Unauthenticated streams work the same way, minus the header. - -### `StreamController` - -Returned by `.stream()` on `TableRef`, `QueryBuilder`, `PipeRef`, and `DLQNamespace` (the DLQ variant is not yet functional server-side — [#197](https://github.com/Wave-RF/WaveHouse/issues/197)). It is **NOT thenable**. - -```ts -const stream = wh.from('clicks').stream({ since: '2026-01-01T00:00:00Z' }); -``` - -### `.subscribe(subscriber)` → `unsubscribe()` - -Callback-based consumption. Returns a cleanup function. - -```ts -const unsub = stream.subscribe({ - next: (event) => { - // event: { table: 'clicks', timestamp: '2026-...', data: { page: '/', ... } } - console.log('New event:', event.data); - }, - status: (state) => { - // state: 'connecting' | 'live' | 'reconnecting' | 'closed' - updateIndicator(state); - }, - error: (err) => { - console.error('Stream error:', err.message); - }, -}); - -// Cleanup — closes the connection if no other subscribers remain -unsub(); -``` - -A handler that throws *during delivery* doesn't end the stream; the exception is logged and the connection keeps running. But delivery of that event stops at the handler that threw — your *later* subscribers, and any concurrent `for await`, do not get it. The first `status` call, the synchronous one `.subscribe()` makes before returning, isn't caught at all and throws back out at you. Wrap your handler bodies in your own `try`/`catch`; see [Error Handling](/sdk/reference#error-handling) for the carve-outs. - -### Async Iterator - -```ts -const stream = wh.from('clicks').stream(); - -for await (const event of stream) { - console.log(event.table, event.data); - if (shouldStop) break; // breaking auto-closes the stream -} -``` - -### `.close()` - -Explicitly close the stream and release all resources. - -```ts -stream.close(); -``` - -### `.status` - -Current connection status: `'connecting' | 'live' | 'reconnecting' | 'closed'`. - -### `.connected(timeoutMs?)` → `Promise` - -Resolves once the stream reaches `live`. Rejects if it is already `closed`, if it closes before connecting, or after `timeoutMs` (default `5000`). Useful when you need the subscription established before doing something that depends on it — inserting a row you expect to see come back, for instance. - -```ts -const stream = wh.from('clicks').stream(); -const unsub = stream.subscribe({ next: (e) => console.log(e) }); -await stream.connected(); // wait until the transport is live -await wh.from('clicks').insert({ page: '/home' }); -``` - -A *timeout* rejection does **not** stop the transport — reconnection is unbounded, so it means "not live yet", not "given up"; call `.close()` if you want it to stop. A rejection because the stream closed is different: there the transport has already stopped. - -One way this rejects against a perfectly healthy stream: if a subscriber you registered *before* calling `.connected()` has a `status` handler that throws, the throw aborts the fan-out before `connected()`'s internal watcher sees `live`, so it times out while `.status` already reads `live`. See [Error Handling](/sdk/reference#error-handling). - -### `StreamOptions` - -| Field | Type | Description | -| ----- | ---- | ----------- | -| `since` | `string` | RFC3339 timestamp for gap-fill replay | -| `signal` | `AbortSignal` | Cancel/close the stream. Wired via `attachSignal()` internally. | - -### `StreamEvent` - -```ts -interface StreamEvent { - table: string; // table name (e.g. 'clicks') - timestamp: string; // received_timestamp (RFC3339Nano) - data: T; // row data -} -``` - -Row values of top-level `DateTime`/`DateTime64` columns inside `data` (not timestamps nested in `Array`/`Map`/`Tuple` columns) arrive in canonical RFC 3339 UTC (`2026-06-21T04:00:00.123Z`), matching what `/v1/query` returns for the same row — `new Date(value)` parses correctly with no zone fix-up. Values WaveHouse couldn't canonicalize (ingest is fail-open) stream in the producer's original spelling, and the `/v1/query` match doesn't hold for them: a spelling ClickHouse accepted anyway still queries back in canonical UTC (one it rejected never lands in the table at all), and a zone-less date-time is what `new Date()` reads as *local* time — though a date-only `YYYY-MM-DD` string is read as UTC, an ECMAScript quirk (see [Timestamp canonicalization](/api#timestamp-canonicalization)). - -### Transport Behavior - -| Transport | Reconnect | Protocol | -| --------- | --------- | -------- | -| SSE over `fetch` | Automatic, jittered backoff, resumes via `Last-Event-ID` | HTTP/2 recommended | - -:::note[SSE connection limit] -The SDK warns above 5 concurrent SSE connections — just under the browser's 6-per-domain limit for HTTP/1.1. HTTP/2 multiplexes over one connection, so the limit doesn't apply there. -::: - -A dropped stream reconnects on a jittered exponential backoff, capped at 30s, and resumes with `Last-Event-ID` so the server gap-fills what was missed. The schedule resets only after a connection has *held* for a few seconds, so a server accepting and immediately closing — slow-consumer eviction, a half-broken upstream — can't pin the client at sub-second retries. Reconnection is **unbounded**: a stream keeps re-dialing until it hits a terminal error or you `close()` it. `options.maxRetries` bounds REST requests only and is never consulted here. - -:::caution[Resumption is at-least-once, and time-bounded] -Delivery across a reconnect is **at-least-once**. The `Last-Event-ID` the client sends is the last event's `received_timestamp`, and the server replays from that instant *inclusively* — so the last event you already saw, and anything sharing its timestamp, arrives again. The SDK does not deduplicate live frames — `liveQuery()` makes one pass at the backfill seam, and only under an ascending order ([#449](https://github.com/Wave-RF/WaveHouse/issues/449)) — so key on `timestamp` plus your own row identity if duplicates matter. - -Replay is also bounded by the server's [`mq.gap_window_minutes`](/configuration#message-queue-nats) — 15 minutes by default. A drop longer than that resumes with a hole and no signal, because the purged messages are simply gone. -::: - -A `4xx` is terminal and surfaces through `error` with the real status code rather than an opaque connection failure — in a browser going cross-origin, only when the rejection passes CORS and the gateway answered whatever preflight the request triggers (`Authorization`, configured `headers`, or `Last-Event-ID` once the stream resumes); otherwise it arrives as a retryable network error instead — indistinguishable from a drop, and retried. It won't be an *authentication* rejection from WaveHouse, which leaves `/v1/stream` ungated and answers an expired token with a filtered view rather than a `401`; the only 4xx it raises itself is `400` for a missing or empty table name, and only on the stream route — a `404` or `405` means the request never reached it, usually a `baseURL` path prefix the proxy didn't strip. Anything else means something in front of it (an auth gateway, a proxy) turned the request away — and note the exception to "retrying wouldn't help": a `429` or `408` from a rate limiter *is* transient, but the stream still ends, so catch it and open a new one after a delay ([#469](https://github.com/Wave-RF/WaveHouse/issues/469)). See [Error Handling](/sdk/reference#error-handling) for every code a stream can report and which ones re-dial. - -Streams go through `options.fetch`, `options.headers`, and `options.fetchOptions` like every other request — which is what lets a stream reach a header-gated origin. A custom `fetch` is asked more of on this path; see [Supplying your own fetch](/sdk#supplying-your-own-fetch). - -### Server-Side Policy Filtering - -Access-control policy applies on the server before anything reaches the client: tables the connection's role can't `select` are skipped, denied columns are stripped from each event, and the role's row-level `filter` is evaluated per subscriber against the connection's JWT claims. A stream on a row-policied table therefore delivers only the rows the policy admits for that connection — and, where the server's in-memory comparison can't prove a match, fewer; see [Access control — where each rule is enforced](/access-control#where-each-rule-is-enforced). Claims are captured when the connection opens: a policy change applies from the next live event (an in-flight gap-fill replay finishes under the policy snapshot taken at connect), while token expiry or claim changes take effect on reconnect. - -### Client-Side Stream Filtering - -On top of that, when a `QueryBuilder` with `.where()` filters or `.select()` columns calls `.stream()`, the returned stream applies those filters client-side: - -```ts -const stream = wh.from('clicks') - .select('page', 'button') - .where('page', '=', '/home') - .stream(); - -// Only events where page === '/home' are emitted, with only page + button columns -``` - -Supported operators: `=`, `!=`, `>`, `>=`, `<`, `<=`, `in`, `like`, `not_like` — the same `FilterOp` set `.where()` takes everywhere (the SDK maps them to wire tokens such as `eq`/`neq` internally). - -Two of them need care, for different reasons. `like` matches **case-insensitively** here, while ClickHouse's `LIKE` on the query path is case-sensitive — so inside a single `liveQuery()` the backfill rows arrive through the server's case-sensitive filter and the live frames through this one, and a pattern like `'/Home%'` can admit live events whose historical counterparts the backfill excluded. `not_like` has no server-side counterpart at all: `/v1/query` rejects it with a `400` (see the operator table in [Queries](/sdk/queries)), so a `liveQuery()` filtered on it gets an error `Result` in `initial()`, drops whatever was buffered during the backfill window, and runs live-only from there. - ---- - -## Live Queries - -Live queries combine a historical backfill (`.fetch()`) with a real-time stream, providing a seamless initial load + live updates experience. - -```ts -const lq = wh.from('clicks') - .selectAll() - .where('page', '=', '/home') - .orderBy('received_timestamp', 'desc') - .limit(100) - .liveQuery({ - initial: (result) => { - // Called once with the historical data - setRows(result.data ?? []); - }, - next: (event) => { - // Called for each live event after backfill - addRow(event.data); - }, - error: (err) => console.error(err), - }); - -// Cleanup -lq.close(); -``` - -### `StreamSubscriber` - -```ts -interface StreamSubscriber { - initial?: (result: Result) => void; // Historical backfill result - next: (event: StreamEvent) => void; // Live events - status?: (state: StreamStatus) => void; // Connection state changes - error?: (err: WaveHouseError) => void; // Errors -} -``` - -### How it works - -1. Opens the stream **immediately** and buffers incoming events. -2. Runs the `.fetch()` query for historical data, calls `subscriber.initial()` with the result — unless the fetch itself throws, in which case neither happens (see below). -3. Deduplicates buffered events against the **last row** of the backfill result — which is the newest only when the query orders ascending. A `desc` query (like the example above) puts the oldest row last, so for live frames the boundary is the oldest timestamp and this step filters nothing. With a `since` gap-fill in flight it works the other way — replay lands in the same buffer, so replayed events at or older than that row are dropped before reaching `next()`. A projection that omits `received_timestamp` skips the pass entirely. Tracked in [#449](https://github.com/Wave-RF/WaveHouse/issues/449). -4. Flushes remaining buffered events and switches to live mode. - -This "stream-first" approach is what closes the window between the fetch and the stream starting — events arriving during the fetch are buffered rather than missed. It holds only when the backfill completes cleanly. - -#### When the backfill doesn't complete - -Here is how it doesn't, and what each case looks like from your subscriber. "Buffered events" means the ones that arrived during the fetch window, which step 4 would otherwise have flushed; the column reports how many of them reach you. - -| What happened | `initial()` | Live events | `error` | Buffered events | -|---|---|---|---|---| -| Backfill returns an error `Result` | fires, `result.error` set | delivered | — | none delivered | -| Your `initial()` throws | fires, then throws | delivered | — | none delivered — the flush never starts | -| Your `next()` throws during the flush | fires normally | delivered, after the flush stops | — | none after the throw | -| `auth` rejects, the stream connects anyway | never fires | delivered | — | none delivered | -| `auth` rejects for a few attempts, then recovers | never fires | delivered once it recovers | `SSE_AUTH_ERROR` per failed attempt | none delivered | -| `auth` keeps rejecting | never fires | none — never reaches `live` | `SSE_AUTH_ERROR` per attempt | none delivered | -| Relative `baseURL` | never fires | none — the stream is terminated too | terminal `SSE_CONNECT_ERROR` | none delivered | -| Your `status` handler throws on the first, synchronous call | never fires | none — buffered where you can't reach them | — | none delivered | - -**The `status`-throw row is the odd one out**, and everything below is written for the others. It is the one case where the backfill never *starts*: the throw escapes `liveQuery()` before the constructor reaches it, so you get no handle back, nothing can `.close()` the stream, and — because the buffering phase never ends — every event accumulates in an object you have no reference to, indefinitely. `next()` is never called at all. Passing `opts.signal` is the only way to stop it; not throwing is the fix. Throwing on any *later* `status` transition is isolated and logged like `next`. See [Error Handling](/sdk/reference#error-handling). - -**What to do about the rest.** Check `result.error` inside `initial()`, keep the flush handlers total, and — if a missing backfill matters — treat `initial()` never firing as its own failure. Where `auth` rejects and the stream connects anyway nothing else will tell you, and where the other rows *do* raise an error it names `auth` or the URL, never the backfill. Re-run the fetch then; you never have to work out which row you hit. Leave it a moment first: events reach a stream from the message queue before the ingest worker lands them in ClickHouse, and its per-table batcher flushes on size or a deadline with the insert still to complete after that (see [Ingest pipeline](/ingest-pipeline)), so an immediate re-fetch can miss the newest rows. - -**Why `auth` splits the way it does.** The backfill makes the *first* `auth()` call and gets exactly one shot — the token is minted above the REST retry loop, so a rejection there is never retried and the whole backfill is gone. The stream calls `auth()` again on every connection attempt and treats the same rejection as transient. That asymmetry is the entire reason a live query can end up running normally with no snapshot behind it, and why nothing announces it: `error` never fires, and the only trace is that `initial()` didn't. That is the case worth guarding against, because it is the one that looks like success. - -**What the live stream can lose.** In every row where the backfill runs at all: nothing, except in one window — events arriving after the stream goes `live` but before the backfill fails are buffered, and the failure discards the buffer. On the `auth` rows that window opens only if the rejection arrives *after* the connection does: the backfill fails as soon as its own `auth()` call rejects, while the stream needs a second `auth()` call resolved **and** a connection opened. Which way that goes depends on your token provider, and you don't need to know — the recovery above covers both. - -Tracked in [#473](https://github.com/Wave-RF/WaveHouse/issues/473). diff --git a/docs/src/content/docs/sdk/streaming.mdx b/docs/src/content/docs/sdk/streaming.mdx new file mode 100644 index 00000000..198afa32 --- /dev/null +++ b/docs/src/content/docs/sdk/streaming.mdx @@ -0,0 +1,492 @@ +--- +title: "SDK Streaming & Live Queries" +description: "Real-time SSE streams, client-side filtering, and backfill-then-live queries in the WaveHouse SDKs." +--- + +import { Tabs, TabItem } from "@astrojs/starlight/components"; + +Real-time consumption: SSE event streams from tables, builders, and pipes, plus live queries that backfill history before going live. Builders and table refs come from [Queries](/sdk/queries). Both SDKs implement the same protocol and mostly the same client-side filtering; the lifecycles differ, and each tab says how. + +## Streaming + +Streams are Server-Sent Events over HTTP. The auth token rides in an `Authorization: Bearer` header — the same as every other request — and is re-read on each connection attempt, so a stream that outlives its token picks up a fresh one. Unauthenticated streams work the same way, minus the header. + +### The stream controller + +Returned by the stream method on a table ref, a query builder, a pipe ref, and the DLQ namespace (the DLQ variant is not yet functional server-side, [#197](https://github.com/Wave-RF/WaveHouse/issues/197)). + + + + +`.stream()` returns a `StreamController`. It is **NOT thenable**. Streams run over `fetch` rather than `EventSource`, which is what lets them carry the auth header. + +```ts +const stream = wh.from('clicks').stream({ since: '2026-01-01T00:00:00Z' }); +``` + + + + +`.Stream(opts)` returns a `*StreamController` immediately; the connection opens in a background goroutine. Frames are parsed over `net/http` with no runtime dependencies. + +```go +stream := wh.From("clicks").Stream(&wavehouse.StreamOptions{ + Since: "2026-01-01T00:00:00Z", +}) +defer stream.Close() +``` + + + + +### Callback subscriptions + +Callback-based consumption. Subscribing returns a cleanup function that removes just that subscriber. + + + + +```ts +const unsub = stream.subscribe({ + next: (event) => { + // event: { table: 'clicks', timestamp: '2026-...', data: { page: '/', ... } } + console.log('New event:', event.data); + }, + status: (state) => { + // state: 'connecting' | 'live' | 'reconnecting' | 'closed' + updateIndicator(state); + }, + error: (err) => { + console.error('Stream error:', err.message); + }, +}); + +// Cleanup: closes the connection if no other subscribers remain +unsub(); +``` + +A handler that throws *during delivery* doesn't end the stream; the exception is logged and the connection keeps running. But delivery of that event stops at the handler that threw: your *later* subscribers, and any concurrent `for await`, do not get it. The first `status` call, the synchronous one `.subscribe()` makes before returning, isn't caught at all and throws back out at you. Wrap your handler bodies in your own `try`/`catch`; see [Error Handling](/sdk/reference#error-handling) for the carve-outs. + + + + +The `Status` callback fires immediately with the current status. + +```go +unsub := stream.Subscribe(&wavehouse.StreamSubscriber{ + Next: func(e wavehouse.StreamEvent) { + // e: {Table: "clicks", Timestamp: "2026-...", Data: map[string]any{"page": "/", ...}} + fmt.Println("New event:", e.Data) + }, + Status: func(s wavehouse.StreamStatus) { + // s: StatusConnecting | StatusLive | StatusReconnecting | StatusClosed + updateIndicator(s) + }, + Error: func(err error) { + fmt.Println("Stream error:", err) + }, +}) + +// Removes this subscriber only; the connection stays open for any others +// and still needs stream.Close() when you're done with the stream itself. +defer unsub() +``` + + + + +### Iterating events + + + + +The controller is an async iterable. + +```ts +const stream = wh.from('clicks').stream(); + +for await (const event of stream) { + console.log(event.table, event.data); + if (shouldStop) break; // breaking auto-closes the stream +} +``` + + + + +`.Events()` returns a read-only channel, closed automatically when the stream shuts down. It is buffered (256 events) and buffers from `.Stream()` onward, so events arriving before the first `Events()` call are not lost. A slow consumer makes the SDK **drop** new events for that channel rather than block the read loop; the first drop logs via `log`, later drops are silent, and `.Subscribe` callbacks fire regardless. + +```go +stream := wh.From("clicks").Stream(nil) +defer stream.Close() + +for event := range stream.Events() { + fmt.Println(event.Table, event.Data) + if shouldStop { + break + } +} +``` + +:::caution[`break` does not close the stream] +Unlike the TypeScript SDK's async iterator, where breaking a `for await` loop closes the connection, breaking a Go `for range stream.Events()` loop only stops consumption: the background goroutine and HTTP connection persist. Always `defer stream.Close()`. +::: + +:::note[`Events()` carries events only] +`Error` and `Status` are delivered exclusively via `.Subscribe(...)`. The channel closes on terminal errors (401/403/404), so pair `Events()` with a subscriber to learn why a stream ended. +::: + + + + +### Closing a stream + + + + +`stream.close()` explicitly closes the stream and releases all resources. + + + + +`stream.Close()` closes the stream and releases its resources. Non-blocking, and safe to call from inside a subscriber callback. + + + + +### Connection status + + + + +`stream.status` is the current status: `'connecting' | 'live' | 'reconnecting' | 'closed'`. + + + + +`stream.Status()` returns the current `StreamStatus`: `StatusConnecting`, `StatusLive`, `StatusReconnecting`, or `StatusClosed`. + + + + +### Waiting for the stream to go live + +Useful when you need the subscription established before doing something that depends on it, such as inserting a row you expect to see come back. + + + + +`.connected(timeoutMs?)` resolves once the stream reaches `live`. Rejects if it is already `closed`, if it closes before connecting, or after `timeoutMs` (default `5000`). + +```ts +const stream = wh.from('clicks').stream(); +const unsub = stream.subscribe({ next: (e) => console.log(e) }); +await stream.connected(); // wait until the transport is live +await wh.from('clicks').insert({ page: '/home' }); +``` + +A *timeout* rejection does **not** stop the transport: reconnection is unbounded, so it means "not live yet", not "given up"; call `.close()` if you want it to stop. A rejection because the stream closed is different: there the transport has already stopped. + +One way this rejects against a perfectly healthy stream: if a subscriber you registered *before* calling `.connected()` has a `status` handler that throws, the throw aborts the fan-out before `connected()`'s internal watcher sees `live`, so it times out while `.status` already reads `live`. See [Error Handling](/sdk/reference#error-handling). + + + + +`.Connected(ctx)` blocks until the stream reaches `StatusLive` or `ctx` is canceled; it returns an error if the stream closes before connecting. The deadline is yours to set: there is no built-in default. + +```go +ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) +defer cancel() +if err := stream.Connected(ctx); err != nil { + log.Fatal(err) +} +``` + + + + +### Stream options + + + + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `since` | `string` | RFC3339 timestamp for gap-fill replay | +| `signal` | `AbortSignal` | Cancel/close the stream. Wired via `attachSignal()` internally. | + + + + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `Since` | `string` | RFC3339 timestamp for gap-fill replay | + +There is no `Signal`/context field: a stream is not canceled by passing a `context.Context` into `.Stream()`. Call `.Close()` instead. + + + + +### The event shape + + + + +```ts +interface StreamEvent { + table: string; // table name (e.g. 'clicks') + timestamp: string; // received_timestamp (RFC3339Nano) + data: T; // row data +} +``` + +Row values of top-level `DateTime`/`DateTime64` columns inside `data` (not timestamps nested in `Array`/`Map`/`Tuple` columns) arrive in canonical RFC 3339 UTC (`2026-06-21T04:00:00.123Z`), matching what `/v1/query` returns for the same row, so `new Date(value)` parses correctly with no zone fix-up. Values WaveHouse couldn't canonicalize (ingest is fail-open) stream in the producer's original spelling, and the `/v1/query` match doesn't hold for them: a spelling ClickHouse accepted anyway still queries back in canonical UTC (one it rejected never lands in the table at all), and a zone-less date-time is what `new Date()` reads as *local* time, though a date-only `YYYY-MM-DD` string is read as UTC, an ECMAScript quirk (see [Timestamp canonicalization](/api#timestamp-canonicalization)). + + + + +```go +type StreamEvent struct { + Table string // table name (e.g. "clicks") + Timestamp string // received_timestamp (RFC3339Nano) + Data map[string]any // row data +} +``` + +Top-level `DateTime`/`DateTime64` values inside `Data` arrive in canonical RFC 3339 UTC, byte-identical to what `/v1/query` renders for the same stored value, because the ingest handler rewrites them before publishing: a live frame and a later query can't disagree on the spelling of an instant. So a value you sent as `2026-06-21T06:00:00.123+02:00` comes back as `2026-06-21T04:00:00.123Z` (same instant, different spelling), and the canonicalization is deliberately fail-open: a value the server can't parse, or whose zone it can't resolve, is published verbatim. See [Timestamp canonicalization](/api#timestamp-canonicalization). + + + + +### Transport behavior + +A dropped stream reconnects on a jittered exponential backoff, capped at 30s, and resumes from the last event it saw so the server gap-fills what was missed. HTTP/2 is recommended. Reconnection is **unbounded**: a stream keeps re-dialing until it hits a terminal error or you close it. The client option that bounds retries applies to REST requests only and is never consulted here. + +:::caution[Resumption is at-least-once, and time-bounded] +Delivery across a reconnect is **at-least-once**. The resume point the client sends is the last event's `received_timestamp`: a `Last-Event-ID` header in TypeScript, a `?since=` query parameter in Go. The server accepts either and replays from that instant *inclusively*, so the last event you already saw, and anything sharing its timestamp, arrives again. Neither SDK deduplicates live frames (a live query makes one pass at the backfill seam only), so key on `timestamp` plus your own row identity if duplicates matter. + +Replay is also bounded by the server's [`mq.gap_window_minutes`](/configuration#message-queue-nats): 15 minutes by default. A drop longer than that resumes with a hole and no signal, because the purged messages are simply gone. +::: + + + + +The backoff schedule resets only after a connection has *held* for a few seconds, so a server accepting and immediately closing — slow-consumer eviction, a half-broken upstream — can't pin the client at sub-second retries. + +:::note[SSE connection limit] +The SDK warns above 5 concurrent SSE connections, just under the browser's 6-per-domain limit for HTTP/1.1. HTTP/2 multiplexes over one connection, so the limit doesn't apply there. +::: + +A `4xx` is terminal and surfaces through `error` with the real status code rather than an opaque connection failure. In a browser going cross-origin, that only holds when the rejection passes CORS and the gateway answered whatever preflight the request triggers (`Authorization`, configured `headers`, or `Last-Event-ID` once the stream resumes); otherwise it arrives as a retryable network error instead, indistinguishable from a drop, and retried. It won't be an *authentication* rejection from WaveHouse, which leaves `/v1/stream` ungated and answers an expired token with a filtered view rather than a `401`; the only 4xx it raises itself is `400` for a missing or empty table name, and only on the stream route: a `404` or `405` means the request never reached it, usually a `baseURL` path prefix the proxy didn't strip. Anything else means something in front of it (an auth gateway, a proxy) turned the request away. Note the exception to "retrying wouldn't help": a `429` or `408` from a rate limiter *is* transient, but the stream still ends, so catch it and open a new one after a delay ([#469](https://github.com/Wave-RF/WaveHouse/issues/469)). See [Error Handling](/sdk/reference#error-handling) for every code a stream can report and which ones re-dial. + +Streams go through `options.fetch`, `options.headers`, and `options.fetchOptions` like every other request, which is what lets a stream reach a header-gated origin. A custom `fetch` is asked more of on this path; see [Supplying your own fetch](/sdk/setup/typescript#supplying-your-own-fetch). + + + + +Reconnect covers transport failures and retryable responses (5xx/429, plus `SSE_AUTH_ERROR` and `SSE_READ_ERROR`). `SSE_PARSE_ERROR` is retryable but does *not* reconnect: the offending frame is dropped and the same connection carries on. Terminal failures fire the `Error` callback, set status `StatusClosed`, and stop: non-retryable HTTP statuses, `SSE_CONNECT_ERROR` (bad `BaseURL`), `SSE_REDIRECT` (a credentialed request was redirected), and `SSE_BAD_CONTENT_TYPE` (a `200` that wasn't an event stream). Every error reaches the callback as a `*wavehouse.Error`, so `errors.As` and `wavehouse.IsRetryable` work on all of them; see the [error-code table](/sdk/reference#error-handling). + +`/v1/stream` is not admin-gated, so WaveHouse itself never answers a stream with `401`; a `401` on a stream came from something in front of it. `Auth` provider errors during (re)connect are retryable (`SSE_AUTH_ERROR`) and reconnects continue (`ClientOptions.MaxRetries` bounds request retries only, not stream reconnects), so call `.Close()` if the provider fails permanently. Auth goes as an `Authorization: Bearer` header on every connection, re-read from `Auth` per attempt ([note in Creating a client](/sdk/setup/go#creating-a-client)). + + + + +### Server-side policy filtering + +Access-control policy applies on the server before anything reaches the client: tables the connection's role can't `select` are skipped, denied columns are stripped from each event, and the role's row-level `filter` is evaluated per subscriber against the connection's JWT claims, on live frames and gap-fill replay alike. Claims are captured when the connection opens: a policy change applies from the next live event (an in-flight gap-fill replay finishes under the policy snapshot taken at connect), while token expiry or claim changes take effect on reconnect. + +Two things follow. **Event-id gaps are normal on a filtered stream**: a gap means a row was withheld, not that a frame was dropped. And **the row filter fails closed**: a comparison the server can't prove (an unresolvable claim, a type it can't compare) withholds the row rather than passing it. See [Access control: where each rule is enforced](/access-control#where-each-rule-is-enforced). + +### Client-side stream filtering + +On top of that, when a query builder carrying filters or a column projection opens a stream, the returned stream applies those filters client-side. + + + + +```ts +const stream = wh.from('clicks') + .select('page', 'button') + .where('page', '=', '/home') + .stream(); + +// Only events where page === '/home' are emitted, with only page + button columns +``` + +Supported operators: `=`, `!=`, `>`, `>=`, `<`, `<=`, `in`, `like`, `not_like`: the same `FilterOp` set `.where()` takes everywhere (the SDK maps them to wire tokens such as `eq`/`neq` internally). + +Two of them need care, for different reasons. `like` matches **case-insensitively** here, while ClickHouse's `LIKE` on the query path is case-sensitive, so inside a single `liveQuery()` the backfill rows arrive through the server's case-sensitive filter and the live frames through this one, and a pattern like `'/Home%'` can admit live events whose historical counterparts the backfill excluded. `not_like` has no server-side counterpart at all: `/v1/query` rejects it with a `400` (see the operator table in [Queries](/sdk/queries#filtering)), so a `liveQuery()` filtered on it gets an error `Result` in `initial()`, drops whatever was buffered during the backfill window, and runs live-only from there. + + + + +```go +stream := wh.From("clicks"). + Select("page", "button"). + Where("page", wavehouse.OpEq, "/home"). + Stream(nil) + +// Only events where page == "/home" are emitted, with only page + button fields +``` + +Supported operators: `OpEq`, `OpNeq`, `OpGt`, `OpGte`, `OpLt`, `OpLte`, `OpIn`, `OpLike`, `OpNotLike`: the `FilterOp` set `.Where()` takes everywhere. `OpLike`/`OpNotLike` use SQL LIKE semantics (`%`, `_`), case-insensitively, and `OpIn` accepts any Go slice type (e.g. `[]string`, `[]int`). + +**How values are compared.** The client-side evaluator mirrors the server's row-filter comparison rules rather than comparing everything as text: + +- **Timestamps compare chronologically.** Since the server canonicalizes every top-level `DateTime`/`DateTime64` value to RFC 3339 UTC before publishing, a payload may read `2026-06-21T04:00:00Z` while your filter constant names the same instant as `2026-06-21T06:00:00+02:00`. Comparing those as text is wrong in both directions: lexically the payload sorts *below* the constant, so `OpGte` would miss a chronologically equal row. Both sides are parsed as instants instead. +- **Only unambiguous spellings count as instants:** RFC 3339 with an explicit offset or `Z`. A zone-less spelling like `2026-06-21 04:00:00` names an instant only relative to the column's declared timezone, which the server reads from the schema and a stream subscriber does not have, so it is not treated as a timestamp. +- **Ordering an instant against a non-instant fails closed.** If one side parses as a timestamp and the other does not, `OpGt`/`OpGte`/`OpLt`/`OpLte` withhold the row rather than falling back to text comparison, which could admit rows the query path excludes. The usual cause is a zone-less filter constant: give it an offset. +- **A missing column equals only `nil`.** A column absent from the payload does not match the string `""`. +- **Numbers compare numerically**, so `9 < 100` as you would expect rather than as text. + +:::caution[Integer precision above 2^53] +Event data decodes through `encoding/json` into `map[string]any`, so JSON numbers arrive as `float64`. An integer column beyond 2^53 has already lost exactness before any filter runs, and the server compares such columns in their exact storage domain, so a client-side filter on a very large `UInt64` can disagree with the server's verdict. Filter on a string or timestamp column instead when exactness at that magnitude matters. +::: + + + + +--- + +## Live Queries + +Live queries combine a historical backfill with a real-time stream, providing a seamless initial load + live updates experience. They exist only on the query builder: there is no table-ref shortcut in either SDK. + + + + +```ts +const lq = wh.from('clicks') + .selectAll() + .where('page', '=', '/home') + .orderBy('received_timestamp', 'desc') + .limit(100) + .liveQuery({ + initial: (result) => { + // Called once with the historical data + setRows(result.data ?? []); + }, + next: (event) => { + // Called for each live event after backfill + addRow(event.data); + }, + error: (err) => console.error(err), + }); + +// Cleanup +lq.close(); +``` + +```ts +interface StreamSubscriber { + initial?: (result: Result) => void; // Historical backfill result + next: (event: StreamEvent) => void; // Live events + status?: (state: StreamStatus) => void; // Connection state changes + error?: (err: WaveHouseError) => void; // Errors +} +``` + + + + +```go +lq := wh.From("clicks"). + SelectAll(). + Where("page", wavehouse.OpEq, "/home"). + OrderBy("received_timestamp", "desc"). + Limit(100). + LiveQuery(&wavehouse.StreamSubscriber{ + Initial: func(rows []map[string]any, err error) { + // Called once with the historical backfill. + setRows(rows) + }, + Next: func(e wavehouse.StreamEvent) { + // Called for each live event after backfill. + addRow(e.Data) + }, + Error: func(err error) { + log.Println(err) + }, + }, nil) + +defer lq.Close() +``` + +```go +type StreamSubscriber struct { + // Initial is called once with historical backfill data (live queries only). + Initial func(rows []map[string]any, err error) + // Next is called for each live event. + Next func(event StreamEvent) + // Status is called when the connection status changes. + Status func(status StreamStatus) + // Error is called on stream errors. + Error func(err error) +} +``` + +:::note[`Initial` is always untyped] +Unlike the TypeScript SDK's `initial: (result: Result) => void`, Go's `LiveQuery` takes no type parameter: `Initial` always receives `[]map[string]any` plus a plain `error`, even where you would use `wavehouse.FetchTyped[Row]` for the same query outside a live query. Decode inside the callback if needed. +::: + +`lq.Close()` shuts down the live query and its underlying stream. Idempotent (guarded by `sync.Once`). + + + + +### How it works + +1. Opens the stream **immediately** and buffers incoming events. +2. Runs the backfill query for historical data and hands the result to the `initial` callback. +3. Deduplicates buffered events against the backfill. +4. Flushes remaining buffered events and switches to live mode. + +This "stream-first" approach is what closes the window between the fetch and the stream starting: events arriving during the fetch are buffered rather than missed. It holds only when the backfill completes cleanly. + + + + +Step 2 is skipped entirely if the fetch itself throws (see below). Step 3 deduplicates against the **last row** of the backfill result, which is the newest only when the query orders ascending. A `desc` query (like the example above) puts the oldest row last, so for live frames the boundary is the oldest timestamp and this step filters nothing. With a `since` gap-fill in flight it works the other way: replay lands in the same buffer, so replayed events at or older than that row are dropped before reaching `next()`. A projection that omits `received_timestamp` skips the pass entirely. Tracked in [#449](https://github.com/Wave-RF/WaveHouse/issues/449). + +#### When the backfill doesn't complete + +Here is how it doesn't, and what each case looks like from your subscriber. "Buffered events" means the ones that arrived during the fetch window, which step 4 would otherwise have flushed; the column reports how many of them reach you. + +| What happened | `initial()` | Live events | `error` | Buffered events | +|---|---|---|---|---| +| Backfill returns an error `Result` | fires, `result.error` set | delivered | n/a | none delivered | +| Your `initial()` throws | fires, then throws | delivered | n/a | none delivered, the flush never starts | +| Your `next()` throws during the flush | fires normally | delivered, after the flush stops | n/a | none after the throw | +| `auth` rejects, the stream connects anyway | never fires | delivered | n/a | none delivered | +| `auth` rejects for a few attempts, then recovers | never fires | delivered once it recovers | `SSE_AUTH_ERROR` per failed attempt | none delivered | +| `auth` keeps rejecting | never fires | none, never reaches `live` | `SSE_AUTH_ERROR` per attempt | none delivered | +| Relative `baseURL` | never fires | none, the stream is terminated too | terminal `SSE_CONNECT_ERROR` | none delivered | +| Your `status` handler throws on the first, synchronous call | never fires | none, buffered where you can't reach them | n/a | none delivered | + +**The `status`-throw row is the odd one out**, and everything below is written for the others. It is the one case where the backfill never *starts*: the throw escapes `liveQuery()` before the constructor reaches it, so you get no handle back, nothing can `.close()` the stream, and — because the buffering phase never ends — every event accumulates in an object you have no reference to, indefinitely. `next()` is never called at all. Passing `opts.signal` is the only way to stop it; not throwing is the fix. Throwing on any *later* `status` transition is isolated and logged like `next`. See [Error Handling](/sdk/reference#error-handling). + +**What to do about the rest.** Check `result.error` inside `initial()`, keep the flush handlers total, and if a missing backfill matters, treat `initial()` never firing as its own failure. Where `auth` rejects and the stream connects anyway nothing else will tell you, and where the other rows *do* raise an error it names `auth` or the URL, never the backfill. Re-run the fetch then; you never have to work out which row you hit. Leave it a moment first: events reach a stream from the message queue before the ingest worker lands them in ClickHouse, and its per-table batcher flushes on size or a deadline with the insert still to complete after that (see [Ingest pipeline](/ingest-pipeline)), so an immediate re-fetch can miss the newest rows. + +**Why `auth` splits the way it does.** The backfill makes the *first* `auth()` call and gets exactly one shot: the token is minted above the REST retry loop, so a rejection there is never retried and the whole backfill is gone. The stream calls `auth()` again on every connection attempt and treats the same rejection as transient. That asymmetry is the entire reason a live query can end up running normally with no snapshot behind it, and why nothing announces it: `error` never fires, and the only trace is that `initial()` didn't. That is the case worth guarding against, because it is the one that looks like success. + +**What the live stream can lose.** In every row where the backfill runs at all: nothing, except in one window: events arriving after the stream goes `live` but before the backfill fails are buffered, and the failure discards the buffer. On the `auth` rows that window opens only if the rejection arrives *after* the connection does: the backfill fails as soon as its own `auth()` call rejects, while the stream needs a second `auth()` call resolved **and** a connection opened. Which way that goes depends on your token provider, and you don't need to know: the recovery above covers both. + +Tracked in [#473](https://github.com/Wave-RF/WaveHouse/issues/473). + + + + +Step 2 runs `.FetchUntyped(ctx)` and then calls `sub.Initial(rows, err)`. Step 3 deduplicates against the maximum `received_timestamp` in the backfill (not necessarily the last row). + +:::caution[Dedup needs `received_timestamp` in the projection] +Dedup relies on `received_timestamp`. `.SelectAll()` (or no projection) includes it; a `.Select(...)` omitting it disables dedup, so events in the overlap window are delivered twice: once via `Initial`, once via `Next`. +::: + +:::caution[`OpLike` matching differs between backfill and live] +Client-side `OpLike` is case-insensitive, but server-side backfills use ClickHouse `LIKE`, which is case-sensitive, so a live query filtering on `OpLike` may exclude rows from the backfill that it includes in the live stream ([#451](https://github.com/Wave-RF/WaveHouse/issues/451)). `OpNotLike` is rejected by `/v1/query` with a `400`, failing the `Initial` callback; see [Queries → Filtering](/sdk/queries#filtering). +::: + +**When the backfill doesn't complete.** `Initial(rows, err)` fires with the error, the events buffered during the fetch window are discarded, and live delivery continues from there, so a live query can end up running normally with no snapshot behind it. Check `err` inside `Initial` and re-run the fetch if a missing backfill matters; leave it a moment first, since events reach a stream from the message queue before the ingest worker lands them in ClickHouse (see [Ingest pipeline](/ingest-pipeline)). + + + diff --git a/docs/src/content/docs/why-wavehouse.md b/docs/src/content/docs/why-wavehouse.md index 6798ded0..ae86ce2a 100644 --- a/docs/src/content/docs/why-wavehouse.md +++ b/docs/src/content/docs/why-wavehouse.md @@ -157,7 +157,7 @@ flowchart TB | Schema validation | Custom code in ingest API | Built in (discovers `system.columns`) | | Row/column access control | Custom middleware or a dedicated service | Built in (Hasura-style, JWT-driven) | | Dead letter queue | Custom retry + dead topic on Kafka | Built in (`WAVEHOUSE_DLQ`) | -| Client SDK | Each team writes one | `@wavehouse/sdk` (TypeScript, one dependency, codegen) | +| Client SDK | Each team writes one | Official [TypeScript and Go clients](/sdk) — query builder, live queries, typed rows | The DIY path works — big teams run it — but the ops cost is not small. You're paying for a Kafka cluster (or Confluent bill), a second service you wrote from scratch, and all the debugging hours when the batching consumer stalls at 3 a.m. @@ -197,7 +197,7 @@ Tinybird wins on "zero ops to start." WaveHouse wins on "own your data plane and | Thundering-herd coalescing | ✗ | Custom | ✓ | ✓ Ristretto + singleflight | | Row/column policies with JWT claims | ✗ | Custom | Tokens only | ✓ Hasura-style | | Named parameterized pipes | ✗ | Custom | ✓ | ✓ stored in NATS KV | -| Type-safe client SDK with codegen | ✗ | Per team | Partial | ✓ `@wavehouse/sdk` | +| Type-safe client SDK | ✗ | Per team | Partial | ✓ TypeScript + Go SDKs | | Cost model | Infra only | Infra + eng time | Per-vCPU SaaS | Infra only | ## Part IV — End-to-end data journey diff --git a/scripts/cov/main.go b/scripts/cov/main.go index f4250bfa..02c88979 100644 --- a/scripts/cov/main.go +++ b/scripts/cov/main.go @@ -59,6 +59,34 @@ const ( // see the ts-total path below. var goSuites = []string{"unit", "integration", "e2e"} +// Go suites rendered and gated like goSuites but never merged into the Go +// total: they come from the nested clients/go module — see the go-sdk +// comment in .testcoverage.yml. +var standaloneGoSuites = []string{"go-sdk"} + +// suiteModuleDir maps a non-root-module suite to the module directory its +// covdata came from. `go tool cover -html` resolves the profile's package +// paths through the module in its working directory, so it must run there. +var suiteModuleDir = map[string]string{"go-sdk": "clients/go"} + +// suiteTarget names the make target that populates a suite, for the +// "did you run it?" hint. Only suites whose target isn't `test-` +// need an entry. One source of truth: every hint in this file routes +// through makeTargetFor, so a target rename touches only this map. +var suiteTarget = map[string]string{ + "go-sdk": "test-sdk-go", + "ts-unit": "test-sdk-ts", + "ts-e2e": "test-e2e", // the orchestrator run, not a target of its own +} + +// makeTargetFor returns the make target that populates the named suite. +func makeTargetFor(suite string) string { + if t, ok := suiteTarget[suite]; ok { + return t + } + return "test-" + suite +} + // TypeScript SDK suites (vitest). ts-unit comes from clients/ts; ts-e2e // from tests/e2e/sdk run with --coverage. Both produce Istanbul-format // coverage-final.json that `cov ts-merge` combines into ts-total. @@ -140,6 +168,16 @@ func main() { if err := merge(cfg); err != nil { fatal("%v", err) } + // merge() only prints the standalone suites; gate them here too, or + // go-sdk-only input would exit 0 with no threshold applied. + for _, s := range standaloneGoSuites { + if !hasCovdata(filepath.Join(root, s, "data")) { + continue + } + if err := renderSuite(cfg, s); err != nil { + fatal("%v", err) + } + } if err := mergeTS(cfg); err != nil { fatal("%v", err) } @@ -205,14 +243,14 @@ func goSuiteCoverage(c *config, suite string) (rows []pkgRow, total, covered int dir := filepath.Join(root, suite) dataDir := filepath.Join(dir, "data") if !hasCovdata(dataDir) { - return nil, 0, 0, "", fmt.Errorf("no covdata in %s — did make test-%s run?", dataDir, suite) + return nil, 0, 0, "", fmt.Errorf("no covdata in %s — did make %s run?", dataDir, makeTargetFor(suite)) } profile := filepath.Join(dir, "coverage.txt") htmlOut = filepath.Join(dir, "coverage.html") if err = sh("go", "tool", "covdata", "textfmt", "-i="+dataDir, "-o", profile); err != nil { return nil, 0, 0, "", err } - if err = sh("go", "tool", "cover", "-html="+profile, "-o", htmlOut); err != nil { + if err = renderHTML(suite, profile, htmlOut); err != nil { return nil, 0, 0, "", err } rows, total, covered, err = parseCoverage(profile, c, c.excludesFor(suite)) @@ -222,6 +260,24 @@ func goSuiteCoverage(c *config, suite string) (rows []pkgRow, total, covered int return rows, total, covered, htmlOut, nil } +// renderHTML turns a textfmt profile into the clickable HTML report, running +// from the suite's own module (hence absolute paths) when it is a nested one. +func renderHTML(suite, profile, htmlOut string) error { + dir, nested := suiteModuleDir[suite] + if !nested { + return sh("go", "tool", "cover", "-html="+profile, "-o", htmlOut) + } + absProfile, err := filepath.Abs(profile) + if err != nil { + return err + } + absHTML, err := filepath.Abs(htmlOut) + if err != nil { + return err + } + return shIn(dir, "go", "tool", "cover", "-html="+absProfile, "-o", absHTML) +} + func renderSuite(c *config, suite string) error { rows, total, covered, htmlOut, err := goSuiteCoverage(c, suite) if err != nil { @@ -247,7 +303,7 @@ func renderSuite(c *config, suite string) error { // "one side legitimately absent" (skip, fine) from "nothing ran at all" // (fail, because the caller expected a gate). func hasAnyCoverage() bool { - for _, s := range goSuites { + for _, s := range slices.Concat(goSuites, standaloneGoSuites) { if hasCovdata(filepath.Join(root, s, "data")) { return true } @@ -269,11 +325,7 @@ func merge(c *config) error { dirs = append(dirs, d) fmt.Printf(" %s✔%s %-13s %s\n", green, reset, s, d) } else { - hint := "test-" + s - if s == "unit" { - hint = "test" - } - fmt.Printf(" %s✗%s %-13s (no covdata; run `make %s` to include)\n", yellow, reset, s, hint) + fmt.Printf(" %s✗%s %-13s (no covdata; run `make %s` to include)\n", yellow, reset, s, makeTargetFor(s)) } } if len(dirs) == 0 { @@ -298,6 +350,13 @@ func merge(c *config) error { for _, s := range goSuites { fmt.Printf(" %s%-13s%s %s\n", cyan, s+":", reset, suitePct(c, s)) } + // Nested-module Go suites: gated on their own, never merged above. + for _, s := range standaloneGoSuites { + if pct := suitePct(c, s); pct != "n/a" { + fmt.Printf(" %s%-13s%s %s %s(separate gate; not in merge above)%s\n", + cyan, s+":", reset, pct, yellow, reset) + } + } // Surface TS SDK coverage alongside the Go total — informational only, // not part of the Go merged number above. `make cov` is the gate. for _, s := range append(tsSuites, "ts-total") { @@ -340,7 +399,7 @@ func merge(c *config) error { // // Layout under tmp/coverage/: // -// ts-unit/coverage-final.json ← `make test-ts` +// ts-unit/coverage-final.json ← `make test-sdk-ts` // ts-e2e/coverage-final.json ← `make test-e2e` // ts-merge-input/ ← scratch dir (both renamed json files) // ts-total/ ← merged JSON + requested reports @@ -424,11 +483,11 @@ func mergeTS(c *config) error { filepath.Join(root, name, "coverage-final.json")) } else { fmt.Printf(" %s✗%s %-9s (no coverage-final.json; run `make %s` to include)\n", - yellow, reset, name, ternary(name == "ts-unit", "test-ts", "test-e2e")) + yellow, reset, name, makeTargetFor(name)) } } if len(merged) == 0 { - fmt.Printf(" %sno TS coverage data — skipping ts-merge (run `make test-ts` and/or `make test-e2e` to populate)%s\n", yellow, reset) + fmt.Printf(" %sno TS coverage data — skipping ts-merge (run `make test-sdk-ts` and/or `make test-e2e` to populate)%s\n", yellow, reset) return nil } @@ -551,6 +610,23 @@ func report(c *config) error { }) } + // --- Nested-module Go suites: own gate, not in the total above --- + for i, s := range standaloneGoSuites { + if !hasCovdata(filepath.Join(root, s, "data")) { + rows = append(rows, reportRow{name: s, pct: "n/a", rule: i == 0}) + continue + } + _, total, covered, html, err := goSuiteCoverage(c, s) + if err != nil { + return err + } + th := thresholdFor(c, s) + rows = append(rows, reportRow{ + name: s, pct: formatPctBare(covered, total), gated: true, thresh: th, + pass: meetsThreshold(covered, total, th), html: html, rule: i == 0, + }) + } + // --- TS suites + merged ts-total --- merged, err := mergeTSArtifacts("html", "json-summary") if err != nil { @@ -693,15 +769,6 @@ func formatPctBare(covered, total int) string { // tsHTML is the vitest/nyc HTML report path for a TS suite. func tsHTML(suite string) string { return filepath.Join(root, suite, "index.html") } -// ternary returns a if cond else b. Used inline to keep the merge log -// branching from sprawling into a 5-line if/else. -func ternary[T any](cond bool, a, b T) T { - if cond { - return a - } - return b -} - // copyFile streams src → dst, creating dst and overwriting if it exists. // Used to stage coverage-final.json files under suite-prefixed names // before nyc merge so the inputs land in one directory. @@ -917,9 +984,13 @@ func meetsThreshold(covered, total, threshold int) bool { // sh runs an external command with stdio wired through. Every call site // passes "go" as the program and a fixed series of "tool", "", // flag, … args; the only variable bits are paths we computed ourselves. +func sh(name string, args ...string) error { return shIn("", name, args...) } + +// shIn is sh with an explicit working directory ("" = inherit ours). // #nosec G204,G702 — name and args are not user input. -func sh(name string, args ...string) error { +func shIn(dir, name string, args ...string) error { cmd := exec.CommandContext(context.Background(), name, args...) + cmd.Dir = dir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr return cmd.Run() diff --git a/tests/conformance/conformance_ts.mjs b/tests/conformance/conformance_ts.mjs new file mode 100644 index 00000000..8db0a291 --- /dev/null +++ b/tests/conformance/conformance_ts.mjs @@ -0,0 +1,272 @@ +#!/usr/bin/env node +/** + * Cross-language wire-format conformance test for the TypeScript SDK. + * + * Replays the shared fixture (clients/go/testdata/wire_cases.json, owned by + * the Go module and also replayed by clients/go/conformance_test.go) and + * verifies the TS SDK produces the same HTTP request: method, path, + * content-type, body. + * + * Run: node tests/conformance/conformance_ts.mjs + * Exit 0 = all pass, exit 1 = failures. + */ + +import { readFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +let createClient; +try { + ({ createClient } = await import(join(__dirname, "../../clients/ts/dist/index.js"))); +} catch (err) { + console.error( + "Cannot load the TypeScript SDK build. Run the SDK build first (e.g. `pnpm --dir clients/ts build`).", + ); + console.error(err.message); + process.exit(1); +} + +const cases = JSON.parse( + readFileSync(join(__dirname, "../../clients/go/testdata/wire_cases.json"), "utf-8"), +); + +let lastCapture = { method: "", path: "", contentType: "", body: "" }; + +// Canned responses, matching the real server's shapes (internal/api/*.go) so +// the SDK never errors on decode. Mirrors the Go harness's handler. +function cannedResponse(url, method, contentType) { + if (url.startsWith("/v1/ops/dlq")) return { tables: {}, total: 0 }; + if (url.startsWith("/v1/ops/schema") && method === "GET") return []; + if (url === "/v1/ops/policy/validate" && method === "POST") return { valid: true }; + if (url.startsWith("/v1/ops/policy") && method === "GET") return { tables: {} }; + if (url.startsWith("/v1/ops/pipes/") && method === "GET") + return { name: "test", sql: "SELECT 1" }; + if (url === "/v1/ops/pipes" && method === "GET") return []; + if (url.startsWith("/v1/ingest")) { + return contentType === "application/x-ndjson" + ? { total: 0, succeeded: 0, failed: 0, duplicates: 0 } + : { ok: true }; + } + if (url === "/v1/health") return { status: "ok" }; + return []; +} + +const server = createServer((req, res) => { + const chunks = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => { + lastCapture = { + method: req.method ?? "", + path: req.url ?? "", + contentType: req.headers["content-type"] ?? "", + body: Buffer.concat(chunks).toString("utf-8"), + }; + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify(cannedResponse(lastCapture.path, lastCapture.method, lastCapture.contentType)), + ); + }); +}); + +await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); +const { port } = server.address(); +const baseURL = `http://127.0.0.1:${port}`; + +// Fixture ops that map one-to-one onto a (column, alias) aggregation method. +// An empty arg falls through to the SDK's own default. +const AGGREGATIONS = new Set(["count", "sum", "avg", "min", "max", "countDistinct"]); + +// applyQueryOps replays a fixture operation chain onto a query builder. +// Fixtures always put select first, so rebuilding on select is safe. +function applyQueryOps(wh, table, operations) { + let q = wh.from(table).select(); + for (const op of operations) { + if (AGGREGATIONS.has(op.method)) { + q = q[op.method](op.args?.[0] || undefined, op.args?.[1] || undefined); + continue; + } + switch (op.method) { + case "select": + q = wh.from(table).select(...op.args); + break; + case "selectAll": + q = q.selectAll(); + break; + case "where": + q = q.where(op.args[0], op.args[1], op.args[2]); + break; + case "aggregate": + q = q.aggregate(op.args[0], op.args[1], op.args[2]); + break; + case "groupBy": + q = q.groupBy(...op.args); + break; + case "orderBy": + q = q.orderBy(op.args[0], op.args[1] || "asc"); + break; + case "limit": + q = q.limit(op.args[0]); + break; + case "timeRange": + q = q.timeRange(op.args[0], op.args[1], op.args[2] || undefined); + break; + case "cacheTTL": + q = q.cacheTTL(op.args[0]); + break; + } + } + return q; +} + +// insertArg returns an ingest case's payload. Hard failure on a malformed +// case, matching the Go harness. +function insertArg(tc) { + const op = tc.operations?.[0]; + if (op?.method !== "insert" || !op.args?.length) { + throw new Error(`${tc.name}: ingest case needs an insert operation with one arg`); + } + return op.args[0]; +} + +// One entry per fixture `endpoint` value. A case naming an endpoint that is +// missing here counts as skipped and fails the run — see below. +const ENDPOINTS = { + query: (wh, tc) => applyQueryOps(wh, tc.table, tc.operations ?? []).fetch(), + ingest: (wh, tc) => wh.from(tc.table).insert(insertArg(tc)), + ingest_batch: (wh, tc) => wh.from(tc.table).insert(insertArg(tc)), + pipe: (wh, tc) => wh.pipe(tc.pipe_name, tc.pipe_params ?? undefined).fetch(), + sql: (wh, tc) => wh.sql(tc.sql), + health: (wh) => wh.sys.health(), + schema_list: (wh) => wh.schema.list(), + schema_refresh: (wh) => wh.schema.refresh(), + policy_get: (wh) => wh.policy.get(), + policy_set: (wh, tc) => wh.policy.set(tc.policy_body), + policy_validate: (wh, tc) => wh.policy.validate(tc.policy_body), + dlq_list: (wh) => wh.dlq.list(), + dlq_table: (wh, tc) => wh.dlq.table(tc.table), + pipes_list: (wh) => wh.pipes.list(), + pipes_get: (wh, tc) => wh.pipes.get(tc.pipe_name), + pipes_set: (wh, tc) => wh.pipes.set(tc.pipe_name, tc.pipe_def), + pipes_delete: (wh, tc) => wh.pipes.delete(tc.pipe_name), +}; + +// Compare request URIs by meaning: same path, same decoded query values, +// regardless of + vs %20 spelling or parameter order (mirrors the Go harness). +function normalizePath(p) { + let u; + try { + u = new URL(p, "http://conformance.invalid"); + } catch { + return p; + } + u.searchParams.sort(); + return `${u.pathname}?${u.searchParams.toString()}`; +} + +function deepEqual(a, b) { + return JSON.stringify(sortKeys(a)) === JSON.stringify(sortKeys(b)); +} + +function sortKeys(v) { + if (v === null || v === undefined) return v; + if (Array.isArray(v)) return v.map(sortKeys); + if (typeof v === "object") { + const sorted = {}; + for (const k of Object.keys(v).sort()) { + sorted[k] = sortKeys(v[k]); + } + return sorted; + } + return v; +} + +let passed = 0; +let failed = 0; +const skippedNames = []; +const failures = []; + +for (const tc of cases) { + lastCapture = { method: "", path: "", contentType: "", body: "" }; + const wh = createClient({ baseURL, options: { maxRetries: 0 } }); + + const run = ENDPOINTS[tc.endpoint]; + if (!run) { + skippedNames.push(`${tc.name} (endpoint: ${tc.endpoint})`); + continue; + } + + try { + await run(wh, tc); + + const errs = []; + for (const [what, want, got] of [ + ["method", tc.expected_method, lastCapture.method], + ["content-type", tc.expected_content_type, lastCapture.contentType], + ]) { + if (want && want !== got) errs.push(`${what}: want ${want}, got ${got}`); + } + if (tc.expected_path && normalizePath(lastCapture.path) !== normalizePath(tc.expected_path)) { + errs.push(`path: want ${tc.expected_path}, got ${lastCapture.path}`); + } + + if (tc.expected_raw_body !== undefined) { + if (lastCapture.body !== tc.expected_raw_body) { + errs.push(`raw body:\n want: ${tc.expected_raw_body}\n got: ${lastCapture.body}`); + } + } else if (tc.expected_body !== undefined && tc.expected_body !== null) { + let captured; + try { + captured = JSON.parse(lastCapture.body); + } catch { + errs.push(`body not valid JSON: ${lastCapture.body}`); + } + if (captured !== undefined && !deepEqual(captured, tc.expected_body)) { + errs.push( + `body mismatch:\n want: ${JSON.stringify(tc.expected_body)}\n got: ${JSON.stringify(captured)}`, + ); + } + } + + if (errs.length > 0) { + failed++; + failures.push({ name: tc.name, errors: errs }); + } else { + passed++; + } + } catch (err) { + failed++; + failures.push({ name: tc.name, errors: [`exception: ${err.message}`] }); + } +} + +server.closeAllConnections?.(); +server.close(); + +const skipped = skippedNames.length; +console.log( + `\nWire-format conformance (TS SDK): ${passed} passed, ${failed} failed, ${skipped} skipped, ${cases.length} total\n`, +); + +for (const name of skippedNames) { + console.log(` - skipped: ${name}`); +} + +for (const f of failures) { + console.log(` ✗ ${f.name}`); + for (const e of f.errors) { + console.log(` ${e}`); + } +} + +if (failed > 0 || skipped > 0 || passed === 0) { + if (passed === 0) console.log(" ✗ nothing ran — every case skipped or the fixture is empty\n"); + if (skipped > 0) + console.log(" ✗ skipped cases break cross-SDK parity — wire up the endpoint above\n"); + process.exit(1); +} else { + console.log(" ✓ All cases passed\n"); + process.exit(0); +}