diff --git a/README.md b/README.md index 583a8aa..993eaab 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ After the test, the result is merged into a single per-(hardware, subject) JSON ## Available tests -### Performance tests (25) +### Performance tests (26) | Test | What it does | | --- | --- | @@ -58,6 +58,7 @@ After the test, the result is merged into a single per-(hardware, subject) JSON | `tcp_to_sns_performance` | TCP in, SNS topic out, observed via an SQS subscription (LocalStack emulator) | | `tcp_to_kinesis_performance` | TCP in, Kinesis stream out across 4 shards (LocalStack emulator) | | `tcp_to_cloudwatch_performance` | TCP in, CloudWatch Logs out (LocalStack emulator) | +| `tcp_to_clickhouse_performance` | TCP in, batch-INSERT into a real ClickHouse server — native protocol (vmetric, otel) or HTTP JSONEachRow (Vector, Fluent Bit, Cribl); receiver counts committed rows | ### Correctness tests (17) @@ -90,11 +91,17 @@ Two caveats when reading the numbers: - **Compare across subjects, not across case types.** The emulators (Python/Node single-process services) cap throughput far below the raw TCP cases. Every subject faces the same ceiling, so rankings are meaningful; absolute EPS is not comparable to `tcp_to_tcp_performance`. - **Out of scope:** Azure Event Hubs, Service Bus, Sentinel and Data Explorer have no usable local emulator; AWS Firehose and DynamoDB delivery are LocalStack pro-tier. Cases for those services would silently measure a mock, so they don't exist. +### ClickHouse ingest case + +`tcp_to_clickhouse_performance` measures **sustained rows/s committed to a real `clickhouse-server` (24.8)** through each subject's native ClickHouse integration. Unlike the emulator cases the target is the real database, and unlike the TCP cases the receiver isn't in the data path — it polls ClickHouse for the committed row count and the harness derives EPS from that timeline. Coverage is the subjects with a first-party ClickHouse path: **vmetric** and the **OpenTelemetry Collector** batch-INSERT over the native TCP protocol (`:9000`); **Vector**, **Fluent Bit** (via its `http` output), and **Cribl Stream** POST JSONEachRow over the HTTP interface (`:8123`). + +Reading the numbers: the generator runs unbounded, so this is a **peak** test. Tools that backpressure cleanly on TCP throttle the generator and deliver nearly everything (low `loss%`); tools that can't shed load once ClickHouse can't keep up, which shows as high `loss%`. Weigh **EPS alongside CPU/memory and loss** — a high raw EPS bought with an order of magnitude more RAM and a large fraction of events dropped is a very different result than the same EPS at a fraction of the resources with almost no loss. + ## Subjects | Name | Image | Version | | --- | --- | --- | -| VirtualMetric DataStream | `vmetric/director` | `2.0.6` | +| VirtualMetric DataStream | `vmetric/director` | `2.0.9` | | Vector | `timberio/vector` | `0.54.0-alpine` | | Fluent Bit | `fluent/fluent-bit` | `5.0` | | Fluentd | `fluent/fluentd` | `v1.17-debian-1` | @@ -127,7 +134,7 @@ PipeBench/ receiver/ Receives output, counts lines, validates correctness collector/ Polls Docker stats API, writes metrics CSV vmetric/ Dockerfile + pre-built binary for the VirtualMetric Director subject - cases/ 51 test cases (27 performance + 24 correctness), each with per-subject configs + cases/ 60 test cases (32 performance + 28 correctness), each with per-subject configs web/ Static PipeBench UI (single HTML + per-(hardware, subject) JSON under web/results/) ``` diff --git a/cases/tcp_to_clickhouse_performance/case.yaml b/cases/tcp_to_clickhouse_performance/case.yaml new file mode 100644 index 0000000..0273fd1 --- /dev/null +++ b/cases/tcp_to_clickhouse_performance/case.yaml @@ -0,0 +1,63 @@ +name: tcp_to_clickhouse_performance +type: performance +description: "Measures throughput of a TCP-in to ClickHouse-out pipeline against a real clickhouse-server (24.8). Unlimited generator rate — pushes peak. Each subject ingests into ClickHouse over its native integration: vmetric and the OpenTelemetry Collector batch-INSERT over the native TCP protocol (:9000, the lowest-overhead wire path); Vector, Fluent Bit, and Cribl Stream POST JSONEachRow over the HTTP interface (:8123). This isolates how efficiently each tool batches and pipelines log ingest into ClickHouse — not the TCP receiver, which is out of the data path here. The receiver polls ClickHouse for the committed row count every 500ms (summing active MergeTree parts across the whole database, so it counts whatever table a subject created) and the harness derives EPS from the row-count timeline." + +duration: 60s +# ClickHouse-server needs a few seconds to open its ports; the receiver creates +# the bench database + table during warmup, before any data reaches the subject. +warmup: 30s +# Sink batches flush on interval/size — give the tail time to land in ClickHouse. +drain_grace: 45s + +endpoints: + - name: clickhouse + image: clickhouse/clickhouse-server:24.8 + env: + CLICKHOUSE_USER: "bench" + CLICKHOUSE_PASSWORD: "benchpass" + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" + # Gate the subject on ClickHouse being up. Most subjects retry their sink + # lazily, but the otel-collector clickhouse exporter creates its schema + # eagerly at startup and crashes if the server isn't accepting connections + # yet — so the subject waits for this healthcheck to pass. + healthcheck: + test: "clickhouse-client --query 'SELECT 1'" + interval: 2s + timeout: 3s + retries: 45 + start_period: 3s + +generator: + mode: tcp + target: "subject:9000" + rate: 0 # unlimited — push as fast as possible + line_size: 256 + format: raw + # Some subjects (Fluent Bit, Cribl) can't backpressure cleanly on TCP when + # ClickHouse can't keep up at peak — they reset the socket instead. Reconnect + # so those resets are absorbed and still yield a measured throughput, rather + # than aborting the whole run. Subjects that backpressure cleanly (vmetric, + # Vector, otel) never trigger it. + reconnect: true + +receiver: + mode: clickhouse + env: + RECEIVER_CH_ENDPOINT: "http://clickhouse:8123" + RECEIVER_CH_USER: "bench" + RECEIVER_CH_PASSWORD: "benchpass" + # Poll the row count twice a second for a tight EPS window. + RECEIVER_CLOUD_POLL_MS: "500" + +requires: [clickhouse_sink] + +subjects: + - vmetric + - vector + - fluent-bit + - otel-collector + - cribl-stream + +configurations: + default: + description: "Raw TCP to a ClickHouse table — no transforms, unlimited rate" diff --git a/cases/tcp_to_clickhouse_performance/configs/cribl-stream/cribl.inited b/cases/tcp_to_clickhouse_performance/configs/cribl-stream/cribl.inited new file mode 100644 index 0000000..e69de29 diff --git a/cases/tcp_to_clickhouse_performance/configs/cribl-stream/cribl.yml b/cases/tcp_to_clickhouse_performance/configs/cribl-stream/cribl.yml new file mode 100644 index 0000000..132066c --- /dev/null +++ b/cases/tcp_to_clickhouse_performance/configs/cribl-stream/cribl.yml @@ -0,0 +1,13 @@ +api: + host: 0.0.0.0 + port: 9999 + disabled: false +auth: + type: local +system: + installType: standalone + intercom: false +workers: + count: -2 + minimum: 1 + memory: 2048 diff --git a/cases/tcp_to_clickhouse_performance/configs/cribl-stream/inputs.yml b/cases/tcp_to_clickhouse_performance/configs/cribl-stream/inputs.yml new file mode 100644 index 0000000..efcb326 --- /dev/null +++ b/cases/tcp_to_clickhouse_performance/configs/cribl-stream/inputs.yml @@ -0,0 +1,16 @@ +inputs: + bench_tcp_in: + type: tcp + host: 0.0.0.0 + port: 9000 + disabled: false + tls: + disabled: true + enableProxyHeader: false + staleChannelFlushMs: 10000 + enableHeader: false + preprocess: + disabled: true + sendToRoutes: true + pqEnabled: false + authType: manual diff --git a/cases/tcp_to_clickhouse_performance/configs/cribl-stream/messages.yml b/cases/tcp_to_clickhouse_performance/configs/cribl-stream/messages.yml new file mode 100644 index 0000000..22e2eb7 --- /dev/null +++ b/cases/tcp_to_clickhouse_performance/configs/cribl-stream/messages.yml @@ -0,0 +1 @@ +messages: {} diff --git a/cases/tcp_to_clickhouse_performance/configs/cribl-stream/outputs.yml b/cases/tcp_to_clickhouse_performance/configs/cribl-stream/outputs.yml new file mode 100644 index 0000000..0f32f7d --- /dev/null +++ b/cases/tcp_to_clickhouse_performance/configs/cribl-stream/outputs.yml @@ -0,0 +1,17 @@ +outputs: + default: + defaultId: bench_clickhouse_out + type: default + bench_clickhouse_out: + type: click_house + url: http://clickhouse:8123 + database: bench + tableName: logs + format: json-each-row + mappingType: automatic + authType: basic + username: bench + password: benchpass + asyncInserts: false + onBackpressure: block + compress: false diff --git a/cases/tcp_to_clickhouse_performance/configs/cribl-stream/pipelines/ch_prep.yml b/cases/tcp_to_clickhouse_performance/configs/cribl-stream/pipelines/ch_prep.yml new file mode 100644 index 0000000..8be8ea0 --- /dev/null +++ b/cases/tcp_to_clickhouse_performance/configs/cribl-stream/pipelines/ch_prep.yml @@ -0,0 +1,16 @@ +id: ch_prep +conf: + output: default + groups: {} + asyncFuncTimeout: 1000 + functions: + - id: eval + filter: "true" + disabled: false + conf: + add: + - name: message + value: _raw + remove: + - "_raw" + - "_time" diff --git a/cases/tcp_to_clickhouse_performance/configs/cribl-stream/pipelines/route.yml b/cases/tcp_to_clickhouse_performance/configs/cribl-stream/pipelines/route.yml new file mode 100644 index 0000000..594e73c --- /dev/null +++ b/cases/tcp_to_clickhouse_performance/configs/cribl-stream/pipelines/route.yml @@ -0,0 +1,12 @@ +id: default +groups: {} +comments: [] +routes: + - id: bench_route + name: bench_clickhouse + final: true + disabled: false + pipeline: ch_prep + filter: "true" + output: bench_clickhouse_out + description: "Map _raw to message, then insert into ClickHouse" diff --git a/cases/tcp_to_clickhouse_performance/configs/fluent-bit.conf b/cases/tcp_to_clickhouse_performance/configs/fluent-bit.conf new file mode 100644 index 0000000..763663d --- /dev/null +++ b/cases/tcp_to_clickhouse_performance/configs/fluent-bit.conf @@ -0,0 +1,43 @@ +# Fluent Bit has no native ClickHouse output; the stock `http` output POSTs +# JSONEachRow to ClickHouse's HTTP interface (:8123) via an INSERT … FORMAT +# JSONEachRow query. The raw TCP line arrives under the `log` key (tcp input, +# Format none); a modify filter renames it to `message` so it lands in the +# `message` column of bench.logs. json_date_key is disabled so the record is a +# clean {"message": "..."} object. + +[SERVICE] + Flush 1 + Log_Level info + Daemon off + +[INPUT] + Name tcp + Listen 0.0.0.0 + Port 9000 + Format none + Chunk_Size 512 + Buffer_Size 1024 + # Bound the in-memory buffer so a slower ClickHouse insert path pauses the + # TCP input (backpressure to the generator) instead of growing until the + # container is OOM-killed at unlimited rate. + Mem_Buf_Limit 256M + +[FILTER] + Name modify + Match * + Rename log message + +[OUTPUT] + Name http + Match * + Host clickhouse + Port 8123 + URI /?query=INSERT+INTO+bench.logs+FORMAT+JSONEachRow&input_format_skip_unknown_fields=1 + Format json_lines + Json_Date_Key false + Header Content-Type application/json + http_user bench + http_passwd benchpass + # Parallel POST workers so the HTTP insert path isn't a single-stream + # bottleneck against ClickHouse. + workers 4 diff --git a/cases/tcp_to_clickhouse_performance/configs/http/vmetric.yml b/cases/tcp_to_clickhouse_performance/configs/http/vmetric.yml new file mode 100644 index 0000000..c2c221b --- /dev/null +++ b/cases/tcp_to_clickhouse_performance/configs/http/vmetric.yml @@ -0,0 +1,30 @@ +# vmetric variant: ClickHouse target over the HTTP interface (:8123) with +# INSERT … FORMAT JSONEachRow + gzip (use_compression) — the same high-throughput +# wire path Vector/Fluent Bit/Cribl use here, for an apples-to-apples comparison +# against the native-protocol default config. Selected with `-c http`. +devices: + - id: 1 + name: tcp-in + type: tcp + properties: + address: "0.0.0.0" + port: 9000 + +targets: + - name: ch-out + type: clickhouse + properties: + address: "clickhouse" + port: 8123 + database: "bench" + table: "logs" + username: "bench" + password: "benchpass" + use_compression: true + +routes: + - name: tcp-to-ch + devices: + - name: tcp-in + targets: + - name: ch-out diff --git a/cases/tcp_to_clickhouse_performance/configs/otel-collector.yaml b/cases/tcp_to_clickhouse_performance/configs/otel-collector.yaml new file mode 100644 index 0000000..faab902 --- /dev/null +++ b/cases/tcp_to_clickhouse_performance/configs/otel-collector.yaml @@ -0,0 +1,40 @@ +# OpenTelemetry Collector (contrib): tcplog receiver in, native `clickhouse` +# exporter out. The exporter INSERTs over the native protocol (:9000) and +# manages its own schema (create_schema) — it writes to its own bench.otel_logs +# table, not the shared bench.logs. The perf receiver counts rows database-wide, +# so otel_logs is measured on equal footing. The body of each log record is the +# raw TCP line. + +receivers: + tcplog: + listen_address: "0.0.0.0:9000" + +processors: + batch: + send_batch_size: 10000 + timeout: 1s + +exporters: + clickhouse: + endpoint: tcp://clickhouse:9000?dial_timeout=10s + database: bench + username: bench + password: benchpass + logs_table_name: otel_logs + create_schema: true + ttl: 0 + timeout: 10s + retry_on_failure: + enabled: true + sending_queue: + queue_size: 1000 + +service: + telemetry: + metrics: + level: none + pipelines: + logs: + receivers: [tcplog] + processors: [batch] + exporters: [clickhouse] diff --git a/cases/tcp_to_clickhouse_performance/configs/parquet/vmetric.yml b/cases/tcp_to_clickhouse_performance/configs/parquet/vmetric.yml new file mode 100644 index 0000000..3249dfa --- /dev/null +++ b/cases/tcp_to_clickhouse_performance/configs/parquet/vmetric.yml @@ -0,0 +1,29 @@ +# vmetric variant: ClickHouse target, columnar Parquet POSTed over the HTTP REST +# interface (:8123). parquet-go encodes a single `message` column file; ClickHouse +# maps it by name. Parquet self-compresses (no gzip). Selected with `-c parquet`. +devices: + - id: 1 + name: tcp-in + type: tcp + properties: + address: "0.0.0.0" + port: 9000 + +targets: + - name: ch-out + type: clickhouse + properties: + address: "clickhouse" + port: 8123 + database: "bench" + table: "logs" + username: "bench" + password: "benchpass" + format: "parquet" + +routes: + - name: tcp-to-ch + devices: + - name: tcp-in + targets: + - name: ch-out diff --git a/cases/tcp_to_clickhouse_performance/configs/profile/vmetric.yml b/cases/tcp_to_clickhouse_performance/configs/profile/vmetric.yml new file mode 100644 index 0000000..8e143b4 --- /dev/null +++ b/cases/tcp_to_clickhouse_performance/configs/profile/vmetric.yml @@ -0,0 +1,35 @@ +# Native ClickHouse target + pprof profiler enabled — used to capture CPU/heap +# profiles of the native ingestion path. Selected with `-c profile`. Reach the +# profiler at http://127.0.0.1:6061/debug/pprof/ from inside the subject +# container (docker exec + wget). +profiler: + status: true + remote: true + director_profile_port: 6061 + +devices: + - id: 1 + name: tcp-in + type: tcp + properties: + address: "0.0.0.0" + port: 9000 + +targets: + - name: ch-out + type: clickhouse + properties: + address: "clickhouse" + port: 9000 + database: "bench" + table: "logs" + username: "bench" + password: "benchpass" + format: "native" + +routes: + - name: tcp-to-ch + devices: + - name: tcp-in + targets: + - name: ch-out diff --git a/cases/tcp_to_clickhouse_performance/configs/tsv/vmetric.yml b/cases/tcp_to_clickhouse_performance/configs/tsv/vmetric.yml new file mode 100644 index 0000000..1bbaea0 --- /dev/null +++ b/cases/tcp_to_clickhouse_performance/configs/tsv/vmetric.yml @@ -0,0 +1,30 @@ +# vmetric variant: ClickHouse target, TabSeparatedRaw over the HTTP interface +# (:8123) + gzip — the raw message bytes go straight into one column with NO JSON +# parsing on the server, the fastest text path. Selected with `-c tsv`. +devices: + - id: 1 + name: tcp-in + type: tcp + properties: + address: "0.0.0.0" + port: 9000 + +targets: + - name: ch-out + type: clickhouse + properties: + address: "clickhouse" + port: 8123 + database: "bench" + table: "logs" + username: "bench" + password: "benchpass" + format: "tabseparatedraw" + use_compression: true + +routes: + - name: tcp-to-ch + devices: + - name: tcp-in + targets: + - name: ch-out diff --git a/cases/tcp_to_clickhouse_performance/configs/vector.toml b/cases/tcp_to_clickhouse_performance/configs/vector.toml new file mode 100644 index 0000000..bcdd83d --- /dev/null +++ b/cases/tcp_to_clickhouse_performance/configs/vector.toml @@ -0,0 +1,32 @@ +# Vector: TCP socket in, native `clickhouse` sink out. The clickhouse sink +# POSTs JSONEachRow batches over the HTTP interface (:8123). The socket source +# with the `bytes` codec puts each raw line into the `message` field, which maps +# to the `message` column of bench.logs; skip_unknown_fields tolerates the +# source's extra metadata fields (timestamp, host, source_type) server-side. + +[sources.in] +type = "socket" +mode = "tcp" +address = "0.0.0.0:9000" +decoding.codec = "bytes" +framing.method = "newline_delimited" + +[sinks.out] +type = "clickhouse" +inputs = ["in"] +endpoint = "http://clickhouse:8123" +database = "bench" +table = "logs" +skip_unknown_fields = true + +[sinks.out.auth] +strategy = "basic" +user = "bench" +password = "benchpass" + +[sinks.out.batch] +max_events = 10000 +timeout_secs = 1 + +[sinks.out.healthcheck] +enabled = false diff --git a/cases/tcp_to_clickhouse_performance/configs/vmetric.yml b/cases/tcp_to_clickhouse_performance/configs/vmetric.yml new file mode 100644 index 0000000..89a7320 --- /dev/null +++ b/cases/tcp_to_clickhouse_performance/configs/vmetric.yml @@ -0,0 +1,30 @@ +# TCP device → ClickHouse target, NATIVE protocol. `format: native` uses the +# clickhouse-go batch INSERT over the native TCP port (9000) — the lowest- +# overhead wire path. The raw message goes into the `message` column of +# bench.logs (created by the harness receiver before data flows). +devices: + - id: 1 + name: tcp-in + type: tcp + properties: + address: "0.0.0.0" + port: 9000 + +targets: + - name: ch-out + type: clickhouse + properties: + address: "clickhouse" + port: 9000 + database: "bench" + table: "logs" + username: "bench" + password: "benchpass" + format: "native" + +routes: + - name: tcp-to-ch + devices: + - name: tcp-in + targets: + - name: ch-out diff --git a/containers/generator/main.go b/containers/generator/main.go index 0e2a222..244637c 100644 --- a/containers/generator/main.go +++ b/containers/generator/main.go @@ -46,6 +46,7 @@ type config struct { Warmup time.Duration Sequenced bool // embed SEQ= in each line for correctness Connections int // parallel TCP/HTTP connections (default 1) + Reconnect bool // redial + resume each parallel connection after a transient break, instead of failing the run (opt-in; sink-bound perf cases where a saturated subject resets the socket) SeqOffset int64 // starting sequence number — set per worker by the parallel dispatcher so global sequences don't overlap across workers (otherwise each worker emits 0..perWorker, breaking the receiver-side dedup check) ConnOffset int // global offset added to connection IDs — set by the harness in multi-generator mode so CONN= values from different generators never collide downstream @@ -329,6 +330,7 @@ func loadConfig() config { if cfg.Connections < 1 { cfg.Connections = 1 } + cfg.Reconnect = getEnvBool("GENERATOR_RECONNECT", false) warmupStr := getEnv("GENERATOR_WARMUP", "5s") w, err := time.ParseDuration(warmupStr) @@ -352,7 +354,15 @@ func loadConfig() config { func dialTCP(target string) (net.Conn, error) { timeout := time.Duration(getEnvInt("GENERATOR_CONNECT_TIMEOUT", 120)) * time.Second - deadline := time.Now().Add(timeout) + return dialTCPDeadline(target, time.Now().Add(timeout)) +} + +// dialTCPDeadline retries the dial until it succeeds or `deadline` passes. The +// reconnect path passes the run ceiling as the deadline so a worker can't sit +// in a 120s connect-retry loop past the end of the run (which would keep the +// generator process alive past the harness's generator-wait timeout when a +// saturated subject stops accepting connections). +func dialTCPDeadline(target string, deadline time.Time) (net.Conn, error) { // KeepAlive lets the OS detect a dead server within ~15s so a wedged // subject surfaces an error promptly instead of blocking until the // run-duration write deadline fires. @@ -370,7 +380,7 @@ func dialTCP(target string) (net.Conn, error) { fmt.Fprintf(os.Stderr, "generator: tcp connect %s: %v (retrying…)\n", target, err) time.Sleep(2 * time.Second) } - return nil, fmt.Errorf("tcp connect %s after %s: %w", target, timeout, err) + return nil, fmt.Errorf("tcp connect %s: %w", target, err) } // dialMaybeTLS dials cfg.Target either as plain TCP (default) or wraps the @@ -379,15 +389,21 @@ func dialTCP(target string) (net.Conn, error) { // auto-writes self-signed material for TLS-enabled runs. Returns the same // net.Conn type so callers don't need to branch on TLS. func dialMaybeTLS(cfg config) (net.Conn, error) { + timeout := time.Duration(getEnvInt("GENERATOR_CONNECT_TIMEOUT", 120)) * time.Second + return dialMaybeTLSDeadline(cfg, time.Now().Add(timeout)) +} + +// dialMaybeTLSDeadline is dialMaybeTLS bounded by an absolute deadline (see +// dialTCPDeadline) — used by the reconnect path so a worker's redial respects +// the run ceiling instead of the full connect timeout. +func dialMaybeTLSDeadline(cfg config, deadline time.Time) (net.Conn, error) { if !cfg.TLS { - return dialTCP(cfg.Target) + return dialTCPDeadline(cfg.Target, deadline) } tlsCfg, err := buildTLSConfig(cfg) if err != nil { return nil, fmt.Errorf("tls config: %w", err) } - timeout := time.Duration(getEnvInt("GENERATOR_CONNECT_TIMEOUT", 120)) * time.Second - deadline := time.Now().Add(timeout) var lastErr error dialer := &net.Dialer{Timeout: 5 * time.Second, KeepAlive: 15 * time.Second} for time.Now().Before(deadline) { @@ -399,7 +415,7 @@ func dialMaybeTLS(cfg config) (net.Conn, error) { fmt.Fprintf(os.Stderr, "generator: tls connect %s: %v (retrying…)\n", cfg.Target, err) time.Sleep(2 * time.Second) } - return nil, fmt.Errorf("tls connect %s after %s: %w", cfg.Target, timeout, lastErr) + return nil, fmt.Errorf("tls connect %s: %w", cfg.Target, lastErr) } // buildTLSConfig assembles a *tls.Config from cfg, falling back to the @@ -605,6 +621,20 @@ func runTCPParallel(cfg config, clock *sendClock) (int64, int64, error) { wg.Add(1) go func(id int, conn net.Conn) { defer wg.Done() + + // Reconnect mode: on a transient break, redial and resume until the + // run ceiling — the resilience runTCPSingle already has. The run's + // verdict is the receiver's delivered count, so a worker's own + // errors are swallowed rather than failing the whole run. + if cfg.Reconnect { + sent, bytes := sendWorkerReconnect(cfg, id+cfg.ConnOffset, clock, conn) + totalLines.Add(sent) + totalBytes.Add(bytes) + fmt.Fprintf(os.Stderr, "generator: connection %d done: lines=%d bytes=%d\n", + id, sent, bytes) + return + } + defer conn.Close() w := bufio.NewWriterSize(conn, 256*1024) @@ -637,6 +667,85 @@ func runTCPParallel(cfg config, clock *sendClock) (int64, int64, error) { return totalLines.Load(), totalBytes.Load(), firstErr } +// sendWorkerReconnect drives one parallel worker with reconnect-on-break, used +// when GENERATOR_RECONNECT is set. It reuses the already-dialed initialConn for +// the first attempt, then redials on any non-timeout break until the run +// ceiling (cfg.Duration + 5s). This mirrors runTCPSingle but for a single +// worker id in the parallel fan-out; errors are swallowed because the run's +// success is judged by the receiver's delivered count, not the generator's. +func sendWorkerReconnect(cfg config, id int, clock *sendClock, initialConn net.Conn) (int64, int64) { + runEnd := time.Now().Add(cfg.Duration + 5*time.Second) + var totalSent, totalBytes int64 + conn := initialConn + + for { + if time.Now().After(runEnd) { + if conn != nil { + _ = conn.Close() + } + return totalSent, totalBytes + } + if conn == nil { + // Bound the redial to the run ceiling so a worker never lingers in + // the 120s connect-retry loop after the run should have ended (a + // saturated subject that stops accepting connections would otherwise + // keep the generator process alive past the harness timeout). + c, err := dialMaybeTLSDeadline(cfg, runEnd) + if err != nil { + if time.Now().After(runEnd) { + return totalSent, totalBytes + } + time.Sleep(500 * time.Millisecond) + continue + } + conn = c + } + _ = conn.SetWriteDeadline(runEnd) + + // Per-attempt config: inner duration fires a few seconds before the + // absolute conn deadline, and TotalLines is decremented so reconnects + // don't overshoot it (matches runTCPSingle). + attempt := cfg + attempt.Duration = time.Until(runEnd) - 5*time.Second + if attempt.Duration <= 0 { + _ = conn.Close() + return totalSent, totalBytes + } + if attempt.TotalLines > 0 { + attempt.TotalLines -= totalSent + if attempt.TotalLines <= 0 { + _ = conn.Close() + return totalSent, totalBytes + } + } + + w := bufio.NewWriterSize(conn, 256*1024) + sent, bytes, werr := sendLinesConn(attempt, id, clock, func(line []byte) error { + _, e := w.Write(line) + return e + }) + ferr := w.Flush() + _ = conn.Close() + conn = nil + totalSent += sent + totalBytes += bytes + + // Absolute/inner deadline fired, or the attempt finished cleanly + // (duration elapsed / TotalLines hit): clean worker exit. + if isDurationTimeout(werr) || isDurationTimeout(ferr) { + return totalSent, totalBytes + } + if werr == nil && ferr == nil { + return totalSent, totalBytes + } + if time.Now().After(runEnd) { + return totalSent, totalBytes + } + // Transient break (reset/broken pipe) before the ceiling — reconnect. + time.Sleep(500 * time.Millisecond) + } +} + // isDurationTimeout reports whether err is the write deadline firing — // i.e. the test ran long enough that we cut the connection off. That's a // successful exit, not a failure. diff --git a/containers/receiver/clickhouse.go b/containers/receiver/clickhouse.go new file mode 100644 index 0000000..5e62d7b --- /dev/null +++ b/containers/receiver/clickhouse.go @@ -0,0 +1,154 @@ +package main + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "time" +) + +// receiveClickHouse measures ClickHouse-sink throughput by polling a row-count +// query over the HTTP interface (:8123). Every subject in the clickhouse case +// writes into the same `bench` database — vmetric over the native TCP protocol +// (:9000), the others via HTTP JSONEachRow or a native exporter — so this +// receiver is deliberately protocol-agnostic: it only counts committed rows. +// +// The receiver also owns table readiness. A performance topology has no init +// container or DDL hook (unlike the correctness ClickHouse driver, which +// creates the table imperatively in Go), so this mode first creates the target +// database + table itself, retrying until clickhouse-server answers, and only +// then starts polling. The generator's warmup delay keeps data from reaching +// the subject until this completes, and every subject retries its sink +// connection regardless — so a not-yet-ready server at startup is harmless. +// +// Each poll records the delta since the previous poll as "received" lines so +// the runner's standard first/last-received EPS window works unchanged — the +// same poll-and-count model the S3/Azure-blob receivers use to drain an object +// store. The default count query sums active MergeTree parts across the whole +// database rather than a single table, so it captures whatever table(s) a +// subject created (bench.logs, otel_logs, …) without the receiver ever needing +// to know the table name a given subject chose. +func receiveClickHouse(cfg config, cnt *counters, val *validator) error { + endpoint := getEnv("RECEIVER_CH_ENDPOINT", "http://clickhouse:8123") + user := getEnv("RECEIVER_CH_USER", "bench") + pass := getEnv("RECEIVER_CH_PASSWORD", "benchpass") + query := getEnv("RECEIVER_CH_QUERY", + "SELECT sum(rows) FROM system.parts WHERE database = 'bench' AND active") + // Rows carry no wire bytes we can observe from a COUNT, so approximate the + // byte throughput from the generator's line size (default matches the raw + // TCP cases' 256-byte payload). Only affects the reported MB/s, not EPS. + bytesPerRow := int64(getEnvInt("RECEIVER_CH_BYTES_PER_ROW", 256)) + + client := &http.Client{Timeout: 15 * time.Second} + + // DDL owned by the receiver — see the function doc. Idempotent so a re-run + // against a persisted volume is a no-op. Ordered: database, then table. + initStmts := []string{ + getEnv("RECEIVER_CH_INIT_DB", "CREATE DATABASE IF NOT EXISTS bench"), + getEnv("RECEIVER_CH_INIT_TABLE", + "CREATE TABLE IF NOT EXISTS bench.logs (message String) ENGINE = MergeTree ORDER BY tuple()"), + } + if err := chInit(client, endpoint, user, pass, initStmts); err != nil { + return err + } + // Best-effort: let JSONEachRow inserts from any subject tolerate columns the + // shared table doesn't declare (a subject that emits {date, log, …} still + // lands a row instead of failing the whole batch). Failure here is + // non-fatal — a subject can set the same option client-side instead. + if stmt := getEnv("RECEIVER_CH_INIT_SETTINGS", + "ALTER USER "+user+" SETTINGS input_format_skip_unknown_fields = 1"); stmt != "" { + if err := chExecHTTP(client, endpoint, user, pass, stmt); err != nil { + fmt.Fprintf(os.Stderr, "receiver: clickhouse settings (non-fatal): %v\n", err) + } + } + + shard := cnt.newShard() + var last int64 + queryURL := endpoint + "/?" + url.Values{"query": {query}}.Encode() + + fmt.Fprintf(os.Stderr, "receiver: polling clickhouse endpoint=%s query=%q\n", endpoint, query) + + return pollLoop("clickhouse", cloudPollInterval(), func(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, queryURL, nil) + if err != nil { + return err + } + // Auth via headers so credentials never appear in the query string + // (which ClickHouse logs). Works on every 22.x+ server. + req.Header.Set("X-ClickHouse-User", user) + req.Header.Set("X-ClickHouse-Key", pass) + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("query: %w", err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("clickhouse http %d: %.200s", resp.StatusCode, string(body)) + } + txt := strings.TrimSpace(string(body)) + if txt == "" { + return nil + } + total, err := strconv.ParseInt(txt, 10, 64) + if err != nil { + return fmt.Errorf("parse count %q: %w", txt, err) + } + // COUNT is monotonic within a run (the volume is fresh each run), so + // record only the increase. Recording deltas is what feeds the runner's + // first/last-received EPS window. + if total > last { + shard.recordN(total-last, (total-last)*bytesPerRow) + last = total + } + return nil + }) +} + +// chInit runs the DDL statements in order, retrying the whole sequence until +// clickhouse-server accepts them or a 2-minute deadline passes. clickhouse- +// server takes a few seconds to open its HTTP port; this is the readiness gate. +func chInit(client *http.Client, endpoint, user, pass string, stmts []string) error { + deadline := time.Now().Add(2 * time.Minute) + var lastErr error + for time.Now().Before(deadline) { + lastErr = nil + for _, s := range stmts { + if err := chExecHTTP(client, endpoint, user, pass, s); err != nil { + lastErr = err + break + } + } + if lastErr == nil { + fmt.Fprintf(os.Stderr, "receiver: clickhouse ready; schema created\n") + return nil + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("clickhouse never became ready: %w", lastErr) +} + +// chExecHTTP POSTs a single statement to the ClickHouse HTTP interface. +func chExecHTTP(client *http.Client, endpoint, user, pass, sql string) error { + req, err := http.NewRequest(http.MethodPost, endpoint+"/", strings.NewReader(sql)) + if err != nil { + return err + } + req.Header.Set("X-ClickHouse-User", user) + req.Header.Set("X-ClickHouse-Key", pass) + resp, err := client.Do(req) + if err != nil { + return err + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("http %d: %.200s", resp.StatusCode, strings.TrimSpace(string(body))) + } + return nil +} diff --git a/containers/receiver/main.go b/containers/receiver/main.go index 071705e..d968451 100644 --- a/containers/receiver/main.go +++ b/containers/receiver/main.go @@ -78,6 +78,24 @@ func (s *connStats) recordLine(byteCount int64) int64 { return n } +// recordN bumps the line/byte counters by a whole batch at once and refreshes +// the receive window. Poll-based modes that learn a row COUNT (clickhouse) +// discover many rows from a single query and can't spin recordLine per row at +// multi-million scale; recordN folds the batch into one pair of atomic adds. +// firstNs is stamped on the first non-empty batch, lastNs on every batch. +func (s *connStats) recordN(lines, bytes int64) { + if lines <= 0 { + return + } + newTotal := s.lines.Add(lines) + s.bytes.Add(bytes) + now := time.Now().UnixNano() + if newTotal == lines { // first batch recorded on this shard + s.firstNs.CompareAndSwap(0, now) + } + s.lastNs.Store(now) +} + // finish records a closing timestamp regardless of where the 1024-line // sampling left off, so the last bucket of lines in a connection isn't // silently dropped from the receive window. @@ -542,6 +560,8 @@ func main() { err = receiveOTLP(cfg, onLine) case "s3": err = receiveS3(cfg, cnt, val) + case "clickhouse": + err = receiveClickHouse(cfg, cnt, val) case "azure_blob": err = receiveAzureBlob(cfg, cnt, val) case "sqs": diff --git a/internal/config/case.go b/internal/config/case.go index ebc4632..7a2d1e8 100644 --- a/internal/config/case.go +++ b/internal/config/case.go @@ -626,6 +626,57 @@ type Endpoint struct { // container; the harness escapes it so docker-compose interpolation leaves // it alone. Command []string `yaml:"command"` + // Healthcheck, when set, attaches a compose healthcheck to the endpoint so + // the subject gates on it (depends_on … condition: service_healthy) before + // starting. Without it the endpoint has no readiness signal and the subject + // starts immediately alongside it. Needed when a subject's sink connects to + // the endpoint eagerly at startup and crashes if it isn't up yet — e.g. the + // otel-collector clickhouse exporter creates its schema on start, so the + // clickhouse endpoint declares a healthcheck and otel waits for it. + Healthcheck *EndpointHealthcheck `yaml:"healthcheck"` +} + +// EndpointHealthcheck is a compose healthcheck for an Endpoint. Only Test is +// required; the timing fields default to a fast poll (see the OrDefault +// helpers) suitable for a local container coming up in a few seconds. +type EndpointHealthcheck struct { + // Test is the command run inside the container, wrapped as CMD-SHELL — a + // non-zero exit means unhealthy. Example (clickhouse-server): + // "clickhouse-client --query 'SELECT 1'". + Test string `yaml:"test"` + // Interval between checks (default "2s"). + Interval string `yaml:"interval"` + // Timeout per check (default "3s"). + Timeout string `yaml:"timeout"` + // Retries before the container is marked unhealthy (default 30). + Retries int `yaml:"retries"` + // StartPeriod grace before failures count (default "2s"). + StartPeriod string `yaml:"start_period"` +} + +func (h *EndpointHealthcheck) IntervalOrDefault() string { + if h != nil && h.Interval != "" { + return h.Interval + } + return "2s" +} +func (h *EndpointHealthcheck) TimeoutOrDefault() string { + if h != nil && h.Timeout != "" { + return h.Timeout + } + return "3s" +} +func (h *EndpointHealthcheck) RetriesOrDefault() int { + if h != nil && h.Retries > 0 { + return h.Retries + } + return 30 +} +func (h *EndpointHealthcheck) StartPeriodOrDefault() string { + if h != nil && h.StartPeriod != "" { + return h.StartPeriod + } + return "2s" } // Rotation parameterizes the director_agent_tls_cert_rotation_correctness @@ -2376,6 +2427,15 @@ type GeneratorConfig struct { LineSize int `yaml:"line_size"` // bytes per line Format string `yaml:"format"` // "raw" | "syslog" | "json" Connections int `yaml:"connections"` // parallel connections (default 1) + // Reconnect, when true, has each parallel TCP connection redial and keep + // sending after a transient break (subject reset/OOM under load) until the + // run ceiling, instead of failing the whole run — the resilience the + // single-connection path already has. Used by sink-bound performance cases + // (e.g. clickhouse) where a subject that can't sustain peak rate resets the + // socket rather than backpressuring cleanly; without it that shows as a run + // ERROR instead of a measured throughput. Default off — every other case + // keeps treating a mid-run break as a hard failure. + Reconnect bool `yaml:"reconnect"` // SampleFile replays a fixed input file instead of synthesizing lines. // Path is relative to the case directory (e.g. "input/sample.cef"). When // set, the generator sends the file's lines verbatim, cycling to reach diff --git a/internal/config/subject.go b/internal/config/subject.go index 9b0dd6a..6c85b19 100644 --- a/internal/config/subject.go +++ b/internal/config/subject.go @@ -123,6 +123,8 @@ var Registry = map[string]Subject{ Capabilities: []string{ "s3_sink", "s3_source", "azure_blob_sink", "sqs_sink", "sns_sink", "kinesis_sink", "cloudwatch_logs_sink", + // Native `clickhouse` sink — HTTP JSONEachRow into a real server. + "clickhouse_sink", }, }, "fluent-bit": { @@ -140,6 +142,10 @@ var Registry = map[string]Subject{ // history. Re-add once upstream ships an insecure-TLS option. Capabilities: []string{ "s3_sink", "azure_blob_sink", + // No native ClickHouse plugin; the stock `http` output POSTs + // JSONEachRow to ClickHouse's HTTP interface (INSERT … FORMAT + // JSONEachRow), which the server ingests like any other client. + "clickhouse_sink", }, }, "fluentd": { @@ -208,7 +214,7 @@ var Registry = map[string]Subject{ "vmetric": { Name: "vmetric", Image: "vmetric/director", - Version: "2.0.6", + Version: "2.0.9", ConfigPath: "/config.yml", // The director resolves device TLS cert_name against its WorkingDir // (/opt/vmetric) and rejects paths outside it, so mount the harness @@ -244,6 +250,9 @@ var Registry = map[string]Subject{ "s3_sink", "s3_source", "azure_blob_sink", "azure_blob_source", "sqs_sink", "sns_sink", "kinesis_sink", "cloudwatch_logs_sink", "s3_avro_sink", "s3_parquet_sink", + // Native `clickhouse` target — batch INSERT over the native TCP + // protocol (:9000), the lowest-overhead wire path. + "clickhouse_sink", }, }, "otel-collector": { @@ -254,7 +263,12 @@ var Registry = map[string]Subject{ // awss3exporter + awscloudwatchlogsexporter take `endpoint`. Note // the s3 exporter writes OTLP-JSON batches, not raw lines — keep // otel out of exact-count correctness cases. - Capabilities: []string{"s3_sink", "cloudwatch_logs_sink"}, + // + // clickhouse: the contrib `clickhouse` exporter INSERTs over the native + // protocol and manages its own schema (create_schema), writing to its + // own otel_logs table rather than the shared bench.logs — the perf + // receiver counts rows database-wide, so it's still measured fairly. + Capabilities: []string{"s3_sink", "cloudwatch_logs_sink", "clickhouse_sink"}, }, "grafana-alloy": { Name: "grafana-alloy", @@ -283,6 +297,9 @@ var Registry = map[string]Subject{ Capabilities: []string{ "s3_sink", "s3_source", "azure_blob_sink", "azure_blob_source", "sqs_sink", "sns_sink", "kinesis_sink", "cloudwatch_logs_sink", + // Native ClickHouse destination — HTTP JSONEachRow with automatic + // field→column mapping. + "clickhouse_sink", }, }, // rotel — Streamfold's Rust-based OTel collector. No config file: diff --git a/internal/orchestrator/docker.go b/internal/orchestrator/docker.go index d2fe7bb..e3fa046 100644 --- a/internal/orchestrator/docker.go +++ b/internal/orchestrator/docker.go @@ -185,8 +185,12 @@ services: image: "{{ .SubjectImage }}" container_name: "{{ .SubjectContainer }}" networks: [bench] -{{- if or .KafkaEnabled .AWSEnabled .AzureEnabled .VaultEnabled .DatabaseEnabled .MinioEnabled .PipelineBrokerEnabled .VerifierLocalDir }} +{{- if or .KafkaEnabled .AWSEnabled .AzureEnabled .VaultEnabled .DatabaseEnabled .MinioEnabled .PipelineBrokerEnabled .VerifierLocalDir .HasHealthcheckEndpoints }} depends_on: +{{- range .HealthcheckEndpoints }} + {{ . }}: + condition: service_healthy +{{- end }} {{- if .VerifierLocalDir }} data-init: condition: service_completed_successfully @@ -441,6 +445,9 @@ services: GENERATOR_WARMUP: "{{ .GenWarmup }}" GENERATOR_SEQUENCED: "{{ .GenSequenced }}" GENERATOR_CONNECTIONS: "{{ .GenConnections }}" +{{- if .GenReconnect }} + GENERATOR_RECONNECT: "true" +{{- end }} {{- if .GenTotalLines }} GENERATOR_TOTAL_LINES: "{{ .GenTotalLines }}" {{- end }} @@ -684,6 +691,14 @@ services: {{- range $k, $v := .Env }} {{ $k }}: "{{ $v }}" {{- end }} +{{- end }} +{{- if .HasHealthcheck }} + healthcheck: + test: ["CMD-SHELL", "{{ .HCTest }}"] + interval: {{ .HCInterval }} + timeout: {{ .HCTimeout }} + retries: {{ .HCRetries }} + start_period: {{ .HCStartPeriod }} {{- end }} restart: "no" {{- end }} @@ -1665,6 +1680,14 @@ type endpointTpl struct { Image string Env map[string]string Command string // pre-formatted YAML list, empty = use the image's default + // Healthcheck (optional): when HasHealthcheck is set, the endpoint gets a + // compose healthcheck and the subject gates on it (service_healthy). + HasHealthcheck bool + HCTest string + HCInterval string + HCTimeout string + HCRetries int + HCStartPeriod string } // clusterNode is one director node service in a director_cluster_correctness run. @@ -1736,6 +1759,7 @@ type composeVars struct { GenWarmup string GenSequenced string GenConnections int + GenReconnect bool GenTotalLines int64 GenEnv map[string]string GenRotateMode string @@ -1923,6 +1947,12 @@ type composeVars struct { // as their own services after the collector. Endpoints []endpointTpl + // HealthcheckEndpoints lists the names of endpoints that declared a + // healthcheck; the subject gates on each (depends_on … service_healthy) so + // it never starts before those auxiliary servers accept connections. + HealthcheckEndpoints []string + HasHealthcheckEndpoints bool + // Agent is the optional external agent container (a case's `agent:` block). // When AgentEnabled is true the template renders an `agent:` service with // depends_on: subject (service_started) so the director's WebSocket + HTTP @@ -2267,6 +2297,7 @@ func writeCompose(path string, cfg RunConfig) error { vars.GenLineSize = genLineSize vars.GenFormat = genFormat vars.GenConnections = resolveGeneratorConnections(g.Connections) + vars.GenReconnect = g.Reconnect vars.GenTotalLines = g.TotalLines vars.GenEnv = mergeEnv(cloudEnvForMode(g.Mode, awsEnv, minioEnv, azureEnv), g.Env) vars.GenRotateMode = g.FileRotation.Mode @@ -2542,12 +2573,25 @@ func writeCompose(path string, cfg RunConfig) error { for k, v := range ep.Env { env[k] = strings.ReplaceAll(v, "$", "$$") } - vars.Endpoints = append(vars.Endpoints, endpointTpl{ + et := endpointTpl{ Name: ep.Name, Image: ep.Image, Env: env, Command: formatYAMLList(cmd), - }) + } + if hc := ep.Healthcheck; hc != nil && hc.Test != "" { + et.HasHealthcheck = true + // Escape "$" so compose doesn't interpolate the shell test (same + // reason as command/env above). + et.HCTest = strings.ReplaceAll(hc.Test, "$", "$$") + et.HCInterval = hc.IntervalOrDefault() + et.HCTimeout = hc.TimeoutOrDefault() + et.HCRetries = hc.RetriesOrDefault() + et.HCStartPeriod = hc.StartPeriodOrDefault() + vars.HealthcheckEndpoints = append(vars.HealthcheckEndpoints, ep.Name) + vars.HasHealthcheckEndpoints = true + } + vars.Endpoints = append(vars.Endpoints, et) } if tc.UsesAgent() {