From 741817f92dccde1c840dd026cb02d93f86d23ef7 Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Tue, 6 Oct 2026 14:46:49 +0800 Subject: [PATCH 1/2] chore: smoke-test example; ClickHouse 26.8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit example/smoke runs pg2iceberg against real Postgres, the Iceberg REST catalog and MinIO under a mixed workload: simulate.py's multi-table transactions plus deletes, insert/update/delete churn in one transaction, rollbacks, key changes, TOAST, a partitioned DEFAULT-identity table, bulk transactions, TRUNCATE and a dropped and re-added column, with external compactions, a SIGKILL restart and snapshot expiry alongside. It then compares Postgres with ClickHouse row by row and checks ClickHouse's count(), exiting non-zero on any difference. ClickHouse 26.2 → 26.8 in example/single and the e2e stack. 26.2's trivial count() answered from the snapshot summary, ignoring equality deletes; 26.8 counts from the manifests and scans when deletes are live. The e2e cases' notes that 26.2 didn't apply equality deletes were stale (their reference outputs show it did) and are gone. example/single's simulator mounted scripts/requirements.txt, which doesn't exist; it installs psycopg2-binary directly now. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 + example/single/docker-compose.yml | 5 +- example/smoke/.gitignore | 1 + example/smoke/README.md | 45 ++++++ example/smoke/compare.py | 61 ++++++++ example/smoke/config.yaml | 35 +++++ example/smoke/docker-compose.yml | 116 +++++++++++++++ example/smoke/run.sh | 64 +++++++++ example/smoke/setup.sql | 13 ++ example/smoke/workload.py | 192 +++++++++++++++++++++++++ tests/cases/00002_update__input.sql | 2 - tests/cases/00003_delete__input.sql | 2 - tests/cases/00004_mixed_ops__input.sql | 2 - tests/docker-compose.yml | 2 +- 14 files changed, 532 insertions(+), 10 deletions(-) create mode 100644 example/smoke/.gitignore create mode 100644 example/smoke/README.md create mode 100644 example/smoke/compare.py create mode 100644 example/smoke/config.yaml create mode 100644 example/smoke/docker-compose.yml create mode 100755 example/smoke/run.sh create mode 100644 example/smoke/setup.sql create mode 100644 example/smoke/workload.py diff --git a/README.md b/README.md index efbd36b..115551a 100644 --- a/README.md +++ b/README.md @@ -230,6 +230,8 @@ SELECT * FROM rideshare.`rideshare.rides` You should see new rows appearing as the simulator drives PG. +To check correctness end to end, [`example/smoke`](example/smoke) runs a mixed workload — deletes, large and rolled-back transactions, key changes, TOAST, schema changes, a crash — against the same stack, then compares Postgres with ClickHouse row by row. + ## Configuration Configuration is YAML-first; see [`config.example.yaml`](config.example.yaml) for the full surface. CLI flags and env vars override individual fields. diff --git a/example/single/docker-compose.yml b/example/single/docker-compose.yml index dc050da..fe66d1a 100644 --- a/example/single/docker-compose.yml +++ b/example/single/docker-compose.yml @@ -114,7 +114,7 @@ services: condition: service_healthy clickhouse: - image: clickhouse/clickhouse-server:26.2 + image: clickhouse/clickhouse-server:26.8 ports: - "8123:8123" environment: @@ -162,9 +162,8 @@ services: DATABASE_URL: postgres://postgres:postgres@postgres:5432/rideshare volumes: - ./simulate.py:/app/simulate.py - - ../../scripts/requirements.txt:/app/requirements.txt working_dir: /app - command: sh -c "pip install -q -r requirements.txt && python simulate.py --interval 5" + command: sh -c "pip install -q psycopg2-binary && python simulate.py --interval 5" volumes: pgdata: diff --git a/example/smoke/.gitignore b/example/smoke/.gitignore new file mode 100644 index 0000000..89f9ac0 --- /dev/null +++ b/example/smoke/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/example/smoke/README.md b/example/smoke/README.md new file mode 100644 index 0000000..a50ff06 --- /dev/null +++ b/example/smoke/README.md @@ -0,0 +1,45 @@ +# Smoke test + +Real Postgres → pg2iceberg → Iceberg REST catalog + MinIO, under a mixed +workload, then a row-by-row comparison of Postgres with ClickHouse reading +the Iceberg tables. Uses [`example/single`](../single)'s rideshare schema, +seed and service configs. + +```sh +cd example/smoke +./run.sh # DURATION=300 by default (seconds of workload) +docker compose --profile workload down -v +``` + +It exits non-zero if Postgres and Iceberg differ. Logs land in `out/`. + +## What runs + +Two concurrent writers ([`workload.py`](workload.py)) mix: + +- `simulate.py`'s rideshare actions: multi-table transactions of inserts and + updates; +- deletes, and transactions that insert, update and delete the same rows; +- rolled-back transactions and primary-key changes; +- TOAST: ~12 KB `drivers.notes` values, and updates that leave them alone. + +`drivers` has Postgres's default replica identity (a delete carries only the +key) and is partitioned by `status`, so a status change moves a row between +partitions ([`setup.sql`](setup.sql), [`config.yaml`](config.yaml)). + +On a timeline, the workload also runs a 5000-row insert, a bulk delete, a +`TRUNCATE`, adding / dropping / re-adding a column, a bulk update and a status +flip of every driver; and [`run.sh`](run.sh) runs `pg2iceberg compact` twice, +SIGKILLs and restarts pg2iceberg, and runs `pg2iceberg maintain` with a 1m +snapshot retention. + +## What's checked + +- [`compare.py`](compare.py): every row and column of each table, Postgres + against ClickHouse (an independent Iceberg reader), plus ClickHouse's + `count()`. +- `pg2iceberg verify`, in `out/verify.log`. +- The replication slot catches up once the workload stops (to within a few + hundred bytes: the coordinator's own state lives in this Postgres). + +On Colima, bind mounts only work under `$HOME`: run from a checkout there. diff --git a/example/smoke/compare.py b/example/smoke/compare.py new file mode 100644 index 0000000..e4e693c --- /dev/null +++ b/example/smoke/compare.py @@ -0,0 +1,61 @@ +"""Compare Postgres with ClickHouse reading the Iceberg tables pg2iceberg +wrote: every row, every column rendered the same way on both sides, and +ClickHouse's `count()`. Exits non-zero on any difference.""" + +import os +import subprocess +import sys +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +TABLES = ["riders", "drivers", "rides", "payments", "ratings"] + + +def pg(sql): + out = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "postgres", "-d", "rideshare", + "-At", "-F", "\t", "-c", sql], + cwd=HERE, capture_output=True, text=True, check=True) + return out.stdout.splitlines() + + +def ch(sql): + req = urllib.request.Request("http://localhost:8123", data=(sql + " FORMAT TSVRaw").encode()) + return urllib.request.urlopen(req).read().decode().splitlines() + + +def render(name, ty): + """The same text for a value on both sides.""" + if name == "notes": # TOAST-sized: compare digests + return f"coalesce(md5({name}), 'NULL')", f"ifNull(lower(hex(MD5({name}))), 'NULL')" + if ty.startswith("timestamp"): + return (f"coalesce((extract(epoch FROM {name}) * 1000000)::bigint::text, 'NULL')", + f"ifNull(toString(toUnixTimestamp64Micro({name})), 'NULL')") + if ty == "numeric": + return f"coalesce(trim_scale({name})::text, 'NULL')", f"ifNull(toString({name}), 'NULL')" + return f"coalesce({name}::text, 'NULL')", f"ifNull(toString({name}), 'NULL')" + + +ok = True +for table in TABLES: + cols = [tuple(r.split("\t")) for r in pg( + "SELECT column_name, data_type FROM information_schema.columns " + f"WHERE table_schema = 'public' AND table_name = '{table}' ORDER BY ordinal_position")] + pg_exprs, ch_exprs = zip(*(render(n, t) for n, t in cols)) + iceberg = f"rideshare.`rideshare.{table}`" + pg_rows = pg(f"SELECT {', '.join(pg_exprs)} FROM {table}") + ch_rows = ch(f"SELECT {', '.join(ch_exprs)} FROM {iceberg}") + ch_count = int(ch(f"SELECT count() FROM {iceberg}")[0]) + only_pg, only_ch = set(pg_rows) - set(ch_rows), set(ch_rows) - set(pg_rows) + dupes = len(ch_rows) - len(set(ch_rows)) + good = not (only_pg or only_ch or dupes) and ch_count == len(pg_rows) + ok &= good + print(f"{table:9} pg={len(pg_rows):6} iceberg={len(ch_rows):6} count()={ch_count:6} " + f"only_pg={len(only_pg)} only_iceberg={len(only_ch)} duplicates={dupes} " + f"{'OK' if good else 'DIFF'}") + for r in sorted(only_pg)[:3]: + print(" pg :", r[:200]) + for r in sorted(only_ch)[:3]: + print(" iceberg :", r[:200]) +print("ALL MATCH" if ok else "MISMATCHES FOUND") +sys.exit(0 if ok else 1) diff --git a/example/smoke/config.yaml b/example/smoke/config.yaml new file mode 100644 index 0000000..b00a3a0 --- /dev/null +++ b/example/smoke/config.yaml @@ -0,0 +1,35 @@ +tables: + - name: public.riders + - name: public.drivers + iceberg: + # A non-key column under DEFAULT replica identity: deletes carry only + # the key, and a status change moves a row between partitions. + partition: ["status"] + - name: public.rides + - name: public.payments + - name: public.ratings + +source: + mode: logical + postgres: + host: postgres + port: 5432 + database: rideshare + user: postgres + password: postgres + logical: + publication_name: pg2iceberg_pub + slot_name: pg2iceberg_slot + +sink: + catalog_uri: http://iceberg-rest:8181 + warehouse: s3://warehouse/ + namespace: rideshare + s3_endpoint: http://minio:9000 + s3_access_key: admin + s3_secret_key: password + s3_region: us-east-1 + flush_interval: 10s + flush_rows: 1000 + +state: {} diff --git a/example/smoke/docker-compose.yml b/example/smoke/docker-compose.yml new file mode 100644 index 0000000..07a1e9b --- /dev/null +++ b/example/smoke/docker-compose.yml @@ -0,0 +1,116 @@ +# Smoke test: real Postgres -> pg2iceberg -> Iceberg REST + MinIO, under a +# mixed workload. Shares example/single's schema, seed and service configs. +# See README.md. +name: pg2iceberg-smoke +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_DB: rideshare + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: ["5502:5432"] + volumes: + - ../single/schema.sql:/docker-entrypoint-initdb.d/01-schema.sql + - ../single/seed.sql:/docker-entrypoint-initdb.d/02-seed.sql + - ./setup.sql:/docker-entrypoint-initdb.d/03-setup.sql + command: ["postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=4", "-c", "max_wal_senders=4"] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d rideshare"] + interval: 2s + timeout: 5s + retries: 30 + + minio: + image: pgsty/minio:RELEASE.2026-08-04T00-00-00Z + command: server /data + environment: + MINIO_ROOT_USER: admin + MINIO_ROOT_PASSWORD: password + ports: ["9000:9000"] + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 2s + timeout: 5s + retries: 15 + + create-bucket: + image: pgsty/mc:RELEASE.2026-09-16T00-00-00Z + depends_on: + minio: {condition: service_healthy} + entrypoint: > + /bin/sh -c "mc alias set m http://minio:9000 admin password && mc mb --ignore-existing m/warehouse" + + iceberg-postgres: + image: postgres:16-alpine + environment: + POSTGRES_DB: iceberg_catalog + POSTGRES_USER: iceberg + POSTGRES_PASSWORD: iceberg + healthcheck: + test: ["CMD-SHELL", "pg_isready -U iceberg"] + interval: 2s + timeout: 5s + retries: 15 + + iceberg-rest: + build: ../single/iceberg-rest + environment: + AWS_ACCESS_KEY_ID: admin + AWS_SECRET_ACCESS_KEY: password + AWS_REGION: us-east-1 + CATALOG_CATALOG__IMPL: org.apache.iceberg.jdbc.JdbcCatalog + CATALOG_URI: jdbc:postgresql://iceberg-postgres:5432/iceberg_catalog + CATALOG_JDBC_USER: iceberg + CATALOG_JDBC_PASSWORD: iceberg + CATALOG_WAREHOUSE: s3://warehouse/ + CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO + CATALOG_S3_ENDPOINT: http://minio:9000 + CATALOG_S3_PATH__STYLE__ACCESS: "true" + ports: ["8181:8181"] + depends_on: + iceberg-postgres: {condition: service_healthy} + create-bucket: {condition: service_completed_successfully} + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:8181/v1/config || exit 1"] + interval: 3s + timeout: 5s + retries: 30 + + pg2iceberg: + build: + context: ../.. + dockerfile: Dockerfile + command: ["run", "--config", "/etc/pg2iceberg/config.yaml"] + environment: + RUST_LOG: info + volumes: + - ./config.yaml:/etc/pg2iceberg/config.yaml:ro + depends_on: + postgres: {condition: service_healthy} + iceberg-rest: {condition: service_healthy} + + clickhouse: + image: clickhouse/clickhouse-server:26.8 + ports: ["8123:8123"] + environment: + AWS_ACCESS_KEY_ID: admin + AWS_SECRET_ACCESS_KEY: password + volumes: + - ../single/clickhouse/config.xml:/etc/clickhouse-server/config.d/s3.xml + - ../single/clickhouse/users.xml:/etc/clickhouse-server/users.d/iceberg.xml + - ../single/clickhouse/init.sql:/docker-entrypoint-initdb.d/init.sql + depends_on: + iceberg-rest: {condition: service_healthy} + + workload: + image: python:3.12-slim + profiles: ["workload"] + environment: + DATABASE_URL: postgres://postgres:postgres@postgres:5432/rideshare + DURATION: ${DURATION:-300} + volumes: + - ../single/simulate.py:/app/simulate.py + - ./workload.py:/app/workload.py + working_dir: /app + command: sh -c "pip install -q psycopg2-binary && python -u workload.py" diff --git a/example/smoke/run.sh b/example/smoke/run.sh new file mode 100755 index 0000000..6325f6d --- /dev/null +++ b/example/smoke/run.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Smoke test: real Postgres -> pg2iceberg -> Iceberg REST + MinIO, under a +# mixed workload (DURATION seconds, default 300) with external compactions, +# a crash and snapshot expiry alongside. Ends with `pg2iceberg verify` and a +# row-by-row comparison of Postgres against ClickHouse reading the Iceberg +# tables. Exits non-zero if they differ. See README.md. +set -u +cd "$(dirname "$0")" +DURATION=${DURATION:-300} +export DURATION +DC="docker compose" +P2I="$DC run --rm -T pg2iceberg" +CFG="--config /etc/pg2iceberg/config.yaml" +mkdir -p out +log() { echo "[$(date +%H:%M:%S)] $*"; } +psql() { $DC exec -T postgres psql -U postgres -d rideshare -At -c "$1"; } + +log "build pg2iceberg" +$DC build -q pg2iceberg || exit 1 + +log "stack up" +$DC up -d postgres minio create-bucket iceberg-postgres iceberg-rest clickhouse +for _ in $(seq 1 90); do + curl -sf http://localhost:8181/v1/config >/dev/null 2>&1 && break; sleep 2 +done +until psql "SELECT 1" >/dev/null 2>&1; do sleep 1; done + +log "pg2iceberg up" +$DC up -d pg2iceberg +sleep 20 + +log "workload start (${DURATION}s)" +$DC --profile workload up -d workload +START=$(date +%s) +at() { while [ $(( $(date +%s) - START )) -lt "$1" ]; do sleep 1; done; } +# Things that happen to pg2iceberg in production, timed into the workload. +step() { + [ "$1" -lt "$DURATION" ] || return 0 + at "$1"; log "$2"; shift 2; "$@"; log " exit $?" +} +step 90 "external compact #1" sh -c "$P2I compact $CFG > out/compact1.log 2>&1" +step 150 "SIGKILL pg2iceberg + restart" sh -c "$DC kill -s SIGKILL pg2iceberg && $DC up -d pg2iceberg" +step 210 "external compact #2" sh -c "$P2I compact $CFG > out/compact2.log 2>&1" +step 240 "maintain (retention 1m)" sh -c "$P2I maintain $CFG --retention 1m > out/maintain.log 2>&1" + +log "waiting for the workload to finish" +while [ -n "$($DC ps -q workload)" ]; do sleep 2; done +$DC --profile workload logs --no-log-prefix workload > out/workload.log 2>&1 +# The coordinator's state lives in this Postgres too, so its own writes +# keep the slot a few hundred bytes behind: wait for it to stop moving. +log "waiting for the slot to catch up" +for _ in $(seq 1 120); do + LAG=$(psql "SELECT pg_wal_lsn_diff(pg_current_wal_lsn(), confirmed_flush_lsn) FROM pg_replication_slots WHERE slot_name = 'pg2iceberg_slot'") + [ "${LAG:-1000000}" -lt 4096 ] && break; sleep 2 +done +log "slot lag: $LAG bytes; letting the materializer drain" +sleep 40 +$DC logs --no-log-prefix pg2iceberg > out/pg2iceberg.log 2>&1 + +log "pg2iceberg verify" +$P2I verify $CFG > out/verify.log 2>&1 +log " exit $?" +log "Postgres vs ClickHouse" +python3 compare.py diff --git a/example/smoke/setup.sql b/example/smoke/setup.sql new file mode 100644 index 0000000..290e3ff --- /dev/null +++ b/example/smoke/setup.sql @@ -0,0 +1,13 @@ +-- Smoke-test extras, after schema + seed. +-- drivers: Postgres's default replica identity (a DELETE/UPDATE's old +-- tuple carries only the key) and a TOAST-able column. +ALTER TABLE drivers REPLICA IDENTITY DEFAULT; +ALTER TABLE drivers ADD COLUMN notes text; +-- More drivers and riders so updates spread out. +INSERT INTO drivers (email, first_name, last_name, phone, city, vehicle_make, vehicle_model, vehicle_year, license_plate, rating, status, signed_up_at) +SELECT 'smoke.driver.' || g || '@example.com', 'D' || g, 'Smoke', '+1-555-9' || g, (ARRAY['San Francisco','New York','Austin'])[1 + g % 3], + 'Toyota', 'Prius', 2024, 'SMK' || g, 4.5, 'active', now() +FROM generate_series(1, 40) g; +INSERT INTO riders (email, first_name, last_name, phone, city, signed_up_at) +SELECT 'smoke.rider.' || g || '@example.com', 'R' || g, 'Smoke', '+1-555-8' || g, (ARRAY['San Francisco','New York','Austin'])[1 + g % 3], now() +FROM generate_series(1, 200) g; diff --git a/example/smoke/workload.py b/example/smoke/workload.py new file mode 100644 index 0000000..778c949 --- /dev/null +++ b/example/smoke/workload.py @@ -0,0 +1,192 @@ +"""Smoke-test workload for pg2iceberg: simulate.py's rideshare actions +(multi-table transactions) plus deletes, churn within a transaction, +rollbacks, key changes, TOAST updates, partition moves, large +transactions, TRUNCATE and schema changes. Two random workers run +concurrently with a timed-events process for DURATION seconds.""" + +import collections +import multiprocessing as mp +import os +import random +import time + +import psycopg2 + +import simulate + +DURATION = float(os.environ.get("DURATION", "300")) +URL = os.environ["DATABASE_URL"] + +# A ~12KB mostly incompressible value: stored out of line (TOAST), so an +# UPDATE that leaves it alone sends pgoutput's "unchanged TOAST" marker. +BIG_NOTE = "(SELECT string_agg(md5(random()::text), '') FROM generate_series(1, 400))" + + +def random_id(cur, table): + cur.execute(f"SELECT id FROM {table} ORDER BY random() LIMIT 1") + row = cur.fetchone() + return row[0] if row else None + + +def delete_rating(conn): + with conn.cursor() as cur: + cur.execute("DELETE FROM ratings WHERE id = (SELECT id FROM ratings ORDER BY random() LIMIT 1)") + conn.commit() + + +def delete_cancelled_ride(conn): + with conn.cursor() as cur: + cur.execute( + "DELETE FROM rides WHERE id = (SELECT r.id FROM rides r WHERE r.status = 'cancelled' " + "AND NOT EXISTS (SELECT 1 FROM payments p WHERE p.ride_id = r.id) " + "AND NOT EXISTS (SELECT 1 FROM ratings g WHERE g.ride_id = r.id) ORDER BY random() LIMIT 1)" + ) + conn.commit() + + +def churn(conn): + """Insert, update and delete one rating, and insert + update another, + in one transaction.""" + with conn.cursor() as cur: + ride = random_id(cur, "rides") + if ride is None: + conn.rollback() + return + cur.execute("INSERT INTO ratings (ride_id, from_rider, score, comment, created_at) " + "VALUES (%s, true, 3, 'churn', now()) RETURNING id", (ride,)) + gone = cur.fetchone()[0] + cur.execute("UPDATE ratings SET score = 4 WHERE id = %s", (gone,)) + cur.execute("DELETE FROM ratings WHERE id = %s", (gone,)) + cur.execute("INSERT INTO ratings (ride_id, from_rider, score, comment, created_at) " + "VALUES (%s, false, 2, 'churn', now()) RETURNING id", (ride,)) + kept = cur.fetchone()[0] + cur.execute("UPDATE ratings SET score = 5, comment = 'churn kept' WHERE id = %s", (kept,)) + conn.commit() + + +def rolled_back(conn): + with conn.cursor() as cur: + ride = random_id(cur, "rides") + if ride is not None: + cur.execute("INSERT INTO ratings (ride_id, from_rider, score, comment, created_at) " + "VALUES (%s, true, 1, 'rolled back', now())", (ride,)) + cur.execute("UPDATE riders SET city = 'Nowhere' WHERE id = (SELECT id FROM riders ORDER BY random() LIMIT 1)") + conn.rollback() + + +def change_key(conn): + with conn.cursor() as cur: + cur.execute("UPDATE ratings SET id = nextval('ratings_id_seq') " + "WHERE id = (SELECT id FROM ratings ORDER BY random() LIMIT 1)") + conn.commit() + + +def toast_write(conn): + with conn.cursor() as cur: + cur.execute(f"UPDATE drivers SET notes = {BIG_NOTE} " + "WHERE id = (SELECT id FROM drivers ORDER BY random() LIMIT 1)") + conn.commit() + + +def toast_untouched(conn): + with conn.cursor() as cur: + cur.execute("UPDATE drivers SET rating = round((4 + random())::numeric, 2) " + "WHERE id = (SELECT id FROM drivers WHERE notes IS NOT NULL ORDER BY random() LIMIT 1)") + conn.commit() + + +def move_partition(conn): + with conn.cursor() as cur: + cur.execute("UPDATE drivers SET status = (ARRAY['active','inactive','suspended'])[1 + floor(random() * 3)::int] " + "WHERE id = (SELECT id FROM drivers ORDER BY random() LIMIT 1)") + conn.commit() + + +def set_tier(conn): + # Fails while the column is dropped; counted as an error. + with conn.cursor() as cur: + cur.execute("UPDATE riders SET tier = (ARRAY['basic','plus','gold'])[1 + floor(random() * 3)::int] " + "WHERE id = (SELECT id FROM riders ORDER BY random() LIMIT 1)") + conn.commit() + + +def simulated(conn): + simulate.pick_action()(conn) + + +MIX = [ + (simulated, 40), (delete_rating, 8), (delete_cancelled_ride, 4), (churn, 8), + (rolled_back, 5), (change_key, 5), (toast_write, 6), (toast_untouched, 10), + (move_partition, 6), (set_tier, 3), +] + + +def worker(seed, deadline, out): + random.seed(seed) + conn = psycopg2.connect(URL) + counts = collections.Counter() + fns, weights = zip(*MIX) + while time.time() < deadline: + fn = random.choices(fns, weights)[0] + try: + fn(conn) + counts[fn.__name__] += 1 + except Exception as e: # noqa: BLE001 — a failed action is just skipped + conn.rollback() + counts[f"error:{fn.__name__}:{type(e).__name__}"] += 1 + time.sleep(random.uniform(0.005, 0.03)) + out.put(dict(counts)) + + +EVENTS = [ + (20, "bulk insert 5000 ratings", + "INSERT INTO ratings (ride_id, from_rider, score, comment, created_at) " + "SELECT (SELECT id FROM rides ORDER BY random() LIMIT 1), true, 5, 'bulk', now() FROM generate_series(1, 5000)"), + (45, "add riders.tier", "ALTER TABLE riders ADD COLUMN tier text DEFAULT 'basic'"), + (70, "bulk delete half the bulk ratings", "DELETE FROM ratings WHERE comment = 'bulk' AND id % 2 = 0"), + (100, "truncate ratings", "TRUNCATE ratings"), + (130, "drop riders.tier", "ALTER TABLE riders DROP COLUMN tier"), + (160, "re-add riders.tier", "ALTER TABLE riders ADD COLUMN tier text"), + (190, "bulk update every ride", "UPDATE rides SET fare_cents = coalesce(fare_cents, 0) + 1"), + (220, "bulk insert 3000 ratings", + "INSERT INTO ratings (ride_id, from_rider, score, comment, created_at) " + "SELECT (SELECT id FROM rides ORDER BY random() LIMIT 1), false, 4, 'bulk2', now() FROM generate_series(1, 3000)"), + (250, "flip every driver's status (partition moves)", + "UPDATE drivers SET status = CASE status WHEN 'active' THEN 'inactive' ELSE 'active' END"), +] + + +def events(start): + conn = psycopg2.connect(URL) + for at, name, sql in EVENTS: + if at >= DURATION: + break + time.sleep(max(0, start + at - time.time())) + t = time.time() + with conn.cursor() as cur: + cur.execute(sql) + n = cur.rowcount + conn.commit() + print(f"[t={at:>3}s] {name}: {n} rows ({time.time() - t:.2f}s)", flush=True) + + +def main(): + start = time.time() + deadline = start + DURATION + out = mp.Queue() + procs = [mp.Process(target=worker, args=(i, deadline, out)) for i in range(2)] + procs.append(mp.Process(target=events, args=(start,))) + for p in procs: + p.start() + total = collections.Counter() + for _ in range(2): + total.update(out.get()) + for p in procs: + p.join() + print("--- workload done ---") + for k, v in sorted(total.items()): + print(f"{k}: {v}") + + +if __name__ == "__main__": + main() diff --git a/tests/cases/00002_update__input.sql b/tests/cases/00002_update__input.sql index ee6c45a..1fe0e45 100644 --- a/tests/cases/00002_update__input.sql +++ b/tests/cases/00002_update__input.sql @@ -11,6 +11,4 @@ INSERT INTO e2e_update (id, name, score) VALUES (2, 'bob', 20), (3, 'charlie', 30); --- NOTE: ClickHouse 26.2 does not apply Iceberg equality deletes yet, --- so the reference output includes both the old and new rows for updated records. UPDATE e2e_update SET score = 99 WHERE id = 2; diff --git a/tests/cases/00003_delete__input.sql b/tests/cases/00003_delete__input.sql index 75c7ae2..8f74559 100644 --- a/tests/cases/00003_delete__input.sql +++ b/tests/cases/00003_delete__input.sql @@ -11,6 +11,4 @@ INSERT INTO e2e_delete (id, name) VALUES (3, 'charlie'), (4, 'diana'); --- NOTE: ClickHouse 26.2 does not apply Iceberg equality deletes yet, --- so the reference output still includes the deleted rows. DELETE FROM e2e_delete WHERE id IN (2, 4); diff --git a/tests/cases/00004_mixed_ops__input.sql b/tests/cases/00004_mixed_ops__input.sql index b12e759..09e8afe 100644 --- a/tests/cases/00004_mixed_ops__input.sql +++ b/tests/cases/00004_mixed_ops__input.sql @@ -13,8 +13,6 @@ INSERT INTO e2e_mixed_ops (id, name, status) VALUES (4, 'diana', 'active'), (5, 'eve', 'active'); --- NOTE: ClickHouse 26.2 does not apply Iceberg equality deletes yet, --- so the reference output includes both old and new rows for updates/deletes. UPDATE e2e_mixed_ops SET status = 'inactive' WHERE id IN (1, 3); DELETE FROM e2e_mixed_ops WHERE id = 5; INSERT INTO e2e_mixed_ops (id, name, status) VALUES (6, 'frank', 'active'); diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index a03e53c..fa691cd 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -84,7 +84,7 @@ services: retries: 10 clickhouse: - image: clickhouse/clickhouse-server:26.2 + image: clickhouse/clickhouse-server:26.8 ports: - "8123:8123" - "9100:9000" From 099587ba846090dff003c0e5700f160cdaea91d5 Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Tue, 6 Oct 2026 16:14:39 +0800 Subject: [PATCH 2/2] chore: make example/single the smoke test example/single is now the smoke-test stack: the mixed workload, run.sh and the Postgres-vs-ClickHouse comparison, with the rideshare schema, seed, simulator, iceberg-rest image and ClickHouse config moved in from the old demo. Its Grafana/Prometheus setup is gone. The README quickstart brings the stack up and starts the workload; ./run.sh runs the full smoke test. example/blue-green built `../../../single/iceberg-rest`, outside the repo; it builds `../single/iceberg-rest` now. Co-Authored-By: Claude Opus 5.5 --- README.md | 5 +- example/blue-green/docker-compose.yml | 2 +- example/{smoke => single}/.gitignore | 0 example/{smoke => single}/README.md | 28 +++-- example/{smoke => single}/compare.py | 0 example/single/config.yaml | 7 +- example/single/docker-compose.yml | 110 +++++------------ .../single/grafana/dashboards/pg2iceberg.json | 1 - .../provisioning/dashboards/default.yml | 11 -- .../provisioning/datasources/prometheus.yml | 8 -- example/single/prometheus.yml | 10 -- example/{smoke => single}/run.sh | 0 example/{smoke => single}/setup.sql | 0 example/{smoke => single}/workload.py | 0 example/smoke/config.yaml | 35 ------ example/smoke/docker-compose.yml | 116 ------------------ 16 files changed, 55 insertions(+), 278 deletions(-) rename example/{smoke => single}/.gitignore (100%) rename example/{smoke => single}/README.md (67%) rename example/{smoke => single}/compare.py (100%) delete mode 120000 example/single/grafana/dashboards/pg2iceberg.json delete mode 100644 example/single/grafana/provisioning/dashboards/default.yml delete mode 100644 example/single/grafana/provisioning/datasources/prometheus.yml delete mode 100644 example/single/prometheus.yml rename example/{smoke => single}/run.sh (100%) rename example/{smoke => single}/setup.sql (100%) rename example/{smoke => single}/workload.py (100%) delete mode 100644 example/smoke/config.yaml delete mode 100644 example/smoke/docker-compose.yml diff --git a/README.md b/README.md index 115551a..94f0356 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,7 @@ Any catalog implementing the [Iceberg REST Catalog spec](https://iceberg.apache. ```sh cd example/single docker compose up -d --wait +docker compose --profile workload up -d workload ``` Open `http://localhost:8123/play` and run: @@ -228,9 +229,9 @@ Open `http://localhost:8123/play` and run: SELECT * FROM rideshare.`rideshare.rides` ``` -You should see new rows appearing as the simulator drives PG. +You should see new rows appearing as the workload drives PG. -To check correctness end to end, [`example/smoke`](example/smoke) runs a mixed workload — deletes, large and rolled-back transactions, key changes, TOAST, schema changes, a crash — against the same stack, then compares Postgres with ClickHouse row by row. +`./run.sh` in the same directory runs it as a smoke test: the mixed workload — deletes, large and rolled-back transactions, key changes, TOAST, schema changes — with a crash, external compactions and snapshot expiry alongside, then a row-by-row comparison of Postgres with ClickHouse. See [`example/single`](example/single). ## Configuration diff --git a/example/blue-green/docker-compose.yml b/example/blue-green/docker-compose.yml index 9503448..58be211 100644 --- a/example/blue-green/docker-compose.yml +++ b/example/blue-green/docker-compose.yml @@ -119,7 +119,7 @@ services: iceberg-rest: # Reuse the same REST catalog image as the single example. - build: ../../../single/iceberg-rest + build: ../single/iceberg-rest environment: AWS_ACCESS_KEY_ID: admin AWS_SECRET_ACCESS_KEY: password diff --git a/example/smoke/.gitignore b/example/single/.gitignore similarity index 100% rename from example/smoke/.gitignore rename to example/single/.gitignore diff --git a/example/smoke/README.md b/example/single/README.md similarity index 67% rename from example/smoke/README.md rename to example/single/README.md index a50ff06..fdb1b51 100644 --- a/example/smoke/README.md +++ b/example/single/README.md @@ -1,17 +1,27 @@ -# Smoke test +# Single-node example and smoke test -Real Postgres → pg2iceberg → Iceberg REST catalog + MinIO, under a mixed -workload, then a row-by-row comparison of Postgres with ClickHouse reading -the Iceberg tables. Uses [`example/single`](../single)'s rideshare schema, -seed and service configs. +A rideshare app's Postgres → pg2iceberg → Iceberg REST catalog + MinIO, with +ClickHouse to query the Iceberg tables and a mixed workload to drive +Postgres. + +Bring it up and watch rows arrive: ```sh -cd example/smoke -./run.sh # DURATION=300 by default (seconds of workload) -docker compose --profile workload down -v +cd example/single +docker compose up -d --wait +docker compose --profile workload up -d workload # DURATION=300 seconds of writes ``` -It exits non-zero if Postgres and Iceberg differ. Logs land in `out/`. +Then query `http://localhost:8123/play`, e.g. +``SELECT * FROM rideshare.`rideshare.rides` ``. + +Or run it as a smoke test, which ends with a row-by-row comparison of +Postgres and ClickHouse and exits non-zero if they differ (logs in `out/`): + +```sh +./run.sh # DURATION=300 by default +docker compose --profile workload down -v +``` ## What runs diff --git a/example/smoke/compare.py b/example/single/compare.py similarity index 100% rename from example/smoke/compare.py rename to example/single/compare.py diff --git a/example/single/config.yaml b/example/single/config.yaml index f685ab0..b00a3a0 100644 --- a/example/single/config.yaml +++ b/example/single/config.yaml @@ -1,20 +1,22 @@ tables: - name: public.riders - name: public.drivers + iceberg: + # A non-key column under DEFAULT replica identity: deletes carry only + # the key, and a status change moves a row between partitions. + partition: ["status"] - name: public.rides - name: public.payments - name: public.ratings source: mode: logical - postgres: host: postgres port: 5432 database: rideshare user: postgres password: postgres - logical: publication_name: pg2iceberg_pub slot_name: pg2iceberg_slot @@ -29,6 +31,5 @@ sink: s3_region: us-east-1 flush_interval: 10s flush_rows: 1000 - flush_bytes: 67108864 state: {} diff --git a/example/single/docker-compose.yml b/example/single/docker-compose.yml index fe66d1a..f1e99a5 100644 --- a/example/single/docker-compose.yml +++ b/example/single/docker-compose.yml @@ -1,3 +1,6 @@ +# Postgres -> pg2iceberg -> Iceberg REST catalog + MinIO, with ClickHouse to +# query the tables and a mixed workload to drive Postgres. See README.md. +name: pg2iceberg-single services: postgres: image: postgres:16-alpine @@ -5,47 +8,37 @@ services: POSTGRES_DB: rideshare POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - ports: - - "5502:5432" + ports: ["5502:5432"] volumes: - ./schema.sql:/docker-entrypoint-initdb.d/01-schema.sql - ./seed.sql:/docker-entrypoint-initdb.d/02-seed.sql - - pgdata:/var/lib/postgresql/data + - ./setup.sql:/docker-entrypoint-initdb.d/03-setup.sql command: ["postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=4", "-c", "max_wal_senders=4"] healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] + test: ["CMD-SHELL", "pg_isready -U postgres -d rideshare"] interval: 2s timeout: 5s - retries: 10 + retries: 30 minio: - # minio/minio + minio/mc are no longer on Docker Hub; pgsty/* is a - # community-maintained drop-in fork (same binaries and entrypoints). image: pgsty/minio:RELEASE.2026-08-04T00-00-00Z - command: server /data --console-address ":9001" + command: server /data environment: MINIO_ROOT_USER: admin MINIO_ROOT_PASSWORD: password - ports: - - "9000:9000" - - "9001:9001" + ports: ["9000:9000"] healthcheck: test: ["CMD", "mc", "ready", "local"] interval: 2s timeout: 5s - retries: 5 + retries: 15 create-bucket: image: pgsty/mc:RELEASE.2026-09-16T00-00-00Z depends_on: - minio: - condition: service_healthy + minio: {condition: service_healthy} entrypoint: > - /bin/sh -c " - mc alias set myminio http://minio:9000 admin password; - mc mb --ignore-existing myminio/warehouse; - exit 0; - " + /bin/sh -c "mc alias set m http://minio:9000 admin password && mc mb --ignore-existing m/warehouse" iceberg-postgres: image: postgres:16-alpine @@ -57,7 +50,7 @@ services: test: ["CMD-SHELL", "pg_isready -U iceberg"] interval: 2s timeout: 5s - retries: 10 + retries: 15 iceberg-rest: build: ./iceberg-rest @@ -73,50 +66,32 @@ services: CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO CATALOG_S3_ENDPOINT: http://minio:9000 CATALOG_S3_PATH__STYLE__ACCESS: "true" - ports: - - "8181:8181" + ports: ["8181:8181"] depends_on: - iceberg-postgres: - condition: service_healthy - create-bucket: - condition: service_completed_successfully + iceberg-postgres: {condition: service_healthy} + create-bucket: {condition: service_completed_successfully} healthcheck: test: ["CMD-SHELL", "curl -sf http://localhost:8181/v1/config || exit 1"] interval: 3s timeout: 5s - retries: 10 + retries: 30 pg2iceberg: build: context: ../.. dockerfile: Dockerfile - # Dockerfile's ENTRYPOINT is the binary; pass the subcommand here. - # Config is sourced from environment variables (see below); no - # `--config` file needed for this example. command: ["run", "--config", "/etc/pg2iceberg/config.yaml"] + environment: + RUST_LOG: info volumes: - ./config.yaml:/etc/pg2iceberg/config.yaml:ro - environment: - POSTGRES_URL: postgresql://postgres:postgres@postgres:5432/rideshare - TABLES: public.riders,public.drivers,public.rides,public.payments,public.ratings - ICEBERG_CATALOG_URL: http://iceberg-rest:8181 - WAREHOUSE: s3://warehouse/ - NAMESPACE: rideshare - S3_ENDPOINT: http://minio:9000 - S3_ACCESS_KEY: admin - S3_SECRET_KEY: password - ports: - - "9091:9090" depends_on: - postgres: - condition: service_healthy - iceberg-rest: - condition: service_healthy + postgres: {condition: service_healthy} + iceberg-rest: {condition: service_healthy} clickhouse: image: clickhouse/clickhouse-server:26.8 - ports: - - "8123:8123" + ports: ["8123:8123"] environment: AWS_ACCESS_KEY_ID: admin AWS_SECRET_ACCESS_KEY: password @@ -125,45 +100,16 @@ services: - ./clickhouse/users.xml:/etc/clickhouse-server/users.d/iceberg.xml - ./clickhouse/init.sql:/docker-entrypoint-initdb.d/init.sql depends_on: - iceberg-rest: - condition: service_healthy - - prometheus: - image: prom/prometheus:v3.4.1 - volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml - ports: - - "9090:9090" - depends_on: - pg2iceberg: - condition: service_started + iceberg-rest: {condition: service_healthy} - grafana: - image: grafana/grafana:11.6.0 - environment: - GF_SECURITY_ADMIN_USER: admin - GF_SECURITY_ADMIN_PASSWORD: admin - GF_AUTH_ANONYMOUS_ENABLED: "true" - GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer - volumes: - - ./grafana/provisioning:/etc/grafana/provisioning - - ./grafana/dashboards:/var/lib/grafana/dashboards - ports: - - "3001:3000" - depends_on: - - prometheus - - simulate: + workload: image: python:3.12-slim - depends_on: - postgres: - condition: service_healthy + profiles: ["workload"] environment: DATABASE_URL: postgres://postgres:postgres@postgres:5432/rideshare + DURATION: ${DURATION:-300} volumes: - ./simulate.py:/app/simulate.py + - ./workload.py:/app/workload.py working_dir: /app - command: sh -c "pip install -q psycopg2-binary && python simulate.py --interval 5" - -volumes: - pgdata: + command: sh -c "pip install -q psycopg2-binary && python -u workload.py" diff --git a/example/single/grafana/dashboards/pg2iceberg.json b/example/single/grafana/dashboards/pg2iceberg.json deleted file mode 120000 index ef4ce80..0000000 --- a/example/single/grafana/dashboards/pg2iceberg.json +++ /dev/null @@ -1 +0,0 @@ -../../../../benchmark/grafana/dashboards/pg2iceberg.json \ No newline at end of file diff --git a/example/single/grafana/provisioning/dashboards/default.yml b/example/single/grafana/provisioning/dashboards/default.yml deleted file mode 100644 index 4825453..0000000 --- a/example/single/grafana/provisioning/dashboards/default.yml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: 1 -providers: - - name: default - orgId: 1 - folder: "" - type: file - disableDeletion: false - editable: true - options: - path: /var/lib/grafana/dashboards - foldersFromFilesStructure: false diff --git a/example/single/grafana/provisioning/datasources/prometheus.yml b/example/single/grafana/provisioning/datasources/prometheus.yml deleted file mode 100644 index 2d43399..0000000 --- a/example/single/grafana/provisioning/datasources/prometheus.yml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: 1 -datasources: - - name: Prometheus - type: prometheus - access: proxy - url: http://prometheus:9090 - isDefault: true - editable: false diff --git a/example/single/prometheus.yml b/example/single/prometheus.yml deleted file mode 100644 index 4f7a62a..0000000 --- a/example/single/prometheus.yml +++ /dev/null @@ -1,10 +0,0 @@ -global: - scrape_interval: 10s - evaluation_interval: 10s - -scrape_configs: - - job_name: pg2iceberg - static_configs: - - targets: ["pg2iceberg:9090"] - labels: - instance: single diff --git a/example/smoke/run.sh b/example/single/run.sh similarity index 100% rename from example/smoke/run.sh rename to example/single/run.sh diff --git a/example/smoke/setup.sql b/example/single/setup.sql similarity index 100% rename from example/smoke/setup.sql rename to example/single/setup.sql diff --git a/example/smoke/workload.py b/example/single/workload.py similarity index 100% rename from example/smoke/workload.py rename to example/single/workload.py diff --git a/example/smoke/config.yaml b/example/smoke/config.yaml deleted file mode 100644 index b00a3a0..0000000 --- a/example/smoke/config.yaml +++ /dev/null @@ -1,35 +0,0 @@ -tables: - - name: public.riders - - name: public.drivers - iceberg: - # A non-key column under DEFAULT replica identity: deletes carry only - # the key, and a status change moves a row between partitions. - partition: ["status"] - - name: public.rides - - name: public.payments - - name: public.ratings - -source: - mode: logical - postgres: - host: postgres - port: 5432 - database: rideshare - user: postgres - password: postgres - logical: - publication_name: pg2iceberg_pub - slot_name: pg2iceberg_slot - -sink: - catalog_uri: http://iceberg-rest:8181 - warehouse: s3://warehouse/ - namespace: rideshare - s3_endpoint: http://minio:9000 - s3_access_key: admin - s3_secret_key: password - s3_region: us-east-1 - flush_interval: 10s - flush_rows: 1000 - -state: {} diff --git a/example/smoke/docker-compose.yml b/example/smoke/docker-compose.yml deleted file mode 100644 index 07a1e9b..0000000 --- a/example/smoke/docker-compose.yml +++ /dev/null @@ -1,116 +0,0 @@ -# Smoke test: real Postgres -> pg2iceberg -> Iceberg REST + MinIO, under a -# mixed workload. Shares example/single's schema, seed and service configs. -# See README.md. -name: pg2iceberg-smoke -services: - postgres: - image: postgres:16-alpine - environment: - POSTGRES_DB: rideshare - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - ports: ["5502:5432"] - volumes: - - ../single/schema.sql:/docker-entrypoint-initdb.d/01-schema.sql - - ../single/seed.sql:/docker-entrypoint-initdb.d/02-seed.sql - - ./setup.sql:/docker-entrypoint-initdb.d/03-setup.sql - command: ["postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=4", "-c", "max_wal_senders=4"] - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres -d rideshare"] - interval: 2s - timeout: 5s - retries: 30 - - minio: - image: pgsty/minio:RELEASE.2026-08-04T00-00-00Z - command: server /data - environment: - MINIO_ROOT_USER: admin - MINIO_ROOT_PASSWORD: password - ports: ["9000:9000"] - healthcheck: - test: ["CMD", "mc", "ready", "local"] - interval: 2s - timeout: 5s - retries: 15 - - create-bucket: - image: pgsty/mc:RELEASE.2026-09-16T00-00-00Z - depends_on: - minio: {condition: service_healthy} - entrypoint: > - /bin/sh -c "mc alias set m http://minio:9000 admin password && mc mb --ignore-existing m/warehouse" - - iceberg-postgres: - image: postgres:16-alpine - environment: - POSTGRES_DB: iceberg_catalog - POSTGRES_USER: iceberg - POSTGRES_PASSWORD: iceberg - healthcheck: - test: ["CMD-SHELL", "pg_isready -U iceberg"] - interval: 2s - timeout: 5s - retries: 15 - - iceberg-rest: - build: ../single/iceberg-rest - environment: - AWS_ACCESS_KEY_ID: admin - AWS_SECRET_ACCESS_KEY: password - AWS_REGION: us-east-1 - CATALOG_CATALOG__IMPL: org.apache.iceberg.jdbc.JdbcCatalog - CATALOG_URI: jdbc:postgresql://iceberg-postgres:5432/iceberg_catalog - CATALOG_JDBC_USER: iceberg - CATALOG_JDBC_PASSWORD: iceberg - CATALOG_WAREHOUSE: s3://warehouse/ - CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO - CATALOG_S3_ENDPOINT: http://minio:9000 - CATALOG_S3_PATH__STYLE__ACCESS: "true" - ports: ["8181:8181"] - depends_on: - iceberg-postgres: {condition: service_healthy} - create-bucket: {condition: service_completed_successfully} - healthcheck: - test: ["CMD-SHELL", "curl -sf http://localhost:8181/v1/config || exit 1"] - interval: 3s - timeout: 5s - retries: 30 - - pg2iceberg: - build: - context: ../.. - dockerfile: Dockerfile - command: ["run", "--config", "/etc/pg2iceberg/config.yaml"] - environment: - RUST_LOG: info - volumes: - - ./config.yaml:/etc/pg2iceberg/config.yaml:ro - depends_on: - postgres: {condition: service_healthy} - iceberg-rest: {condition: service_healthy} - - clickhouse: - image: clickhouse/clickhouse-server:26.8 - ports: ["8123:8123"] - environment: - AWS_ACCESS_KEY_ID: admin - AWS_SECRET_ACCESS_KEY: password - volumes: - - ../single/clickhouse/config.xml:/etc/clickhouse-server/config.d/s3.xml - - ../single/clickhouse/users.xml:/etc/clickhouse-server/users.d/iceberg.xml - - ../single/clickhouse/init.sql:/docker-entrypoint-initdb.d/init.sql - depends_on: - iceberg-rest: {condition: service_healthy} - - workload: - image: python:3.12-slim - profiles: ["workload"] - environment: - DATABASE_URL: postgres://postgres:postgres@postgres:5432/rideshare - DURATION: ${DURATION:-300} - volumes: - - ../single/simulate.py:/app/simulate.py - - ./workload.py:/app/workload.py - working_dir: /app - command: sh -c "pip install -q psycopg2-binary && python -u workload.py"