diff --git a/README.md b/README.md index efbd36b..94f0356 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,7 @@ Any catalog implementing the [Iceberg REST Catalog spec](https://iceberg.apache. ```sh cd example/single docker compose up -d --wait +docker compose --profile workload up -d workload ``` Open `http://localhost:8123/play` and run: @@ -228,7 +229,9 @@ Open `http://localhost:8123/play` and run: SELECT * FROM rideshare.`rideshare.rides` ``` -You should see new rows appearing as the simulator drives PG. +You should see new rows appearing as the workload drives PG. + +`./run.sh` in the same directory runs it as a smoke test: the mixed workload — deletes, large and rolled-back transactions, key changes, TOAST, schema changes — with a crash, external compactions and snapshot expiry alongside, then a row-by-row comparison of Postgres with ClickHouse. See [`example/single`](example/single). ## Configuration diff --git a/example/blue-green/docker-compose.yml b/example/blue-green/docker-compose.yml index 9503448..58be211 100644 --- a/example/blue-green/docker-compose.yml +++ b/example/blue-green/docker-compose.yml @@ -119,7 +119,7 @@ services: iceberg-rest: # Reuse the same REST catalog image as the single example. - build: ../../../single/iceberg-rest + build: ../single/iceberg-rest environment: AWS_ACCESS_KEY_ID: admin AWS_SECRET_ACCESS_KEY: password diff --git a/example/single/.gitignore b/example/single/.gitignore new file mode 100644 index 0000000..89f9ac0 --- /dev/null +++ b/example/single/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/example/single/README.md b/example/single/README.md new file mode 100644 index 0000000..fdb1b51 --- /dev/null +++ b/example/single/README.md @@ -0,0 +1,55 @@ +# Single-node example and smoke test + +A rideshare app's Postgres → pg2iceberg → Iceberg REST catalog + MinIO, with +ClickHouse to query the Iceberg tables and a mixed workload to drive +Postgres. + +Bring it up and watch rows arrive: + +```sh +cd example/single +docker compose up -d --wait +docker compose --profile workload up -d workload # DURATION=300 seconds of writes +``` + +Then query `http://localhost:8123/play`, e.g. +``SELECT * FROM rideshare.`rideshare.rides` ``. + +Or run it as a smoke test, which ends with a row-by-row comparison of +Postgres and ClickHouse and exits non-zero if they differ (logs in `out/`): + +```sh +./run.sh # DURATION=300 by default +docker compose --profile workload down -v +``` + +## What runs + +Two concurrent writers ([`workload.py`](workload.py)) mix: + +- `simulate.py`'s rideshare actions: multi-table transactions of inserts and + updates; +- deletes, and transactions that insert, update and delete the same rows; +- rolled-back transactions and primary-key changes; +- TOAST: ~12 KB `drivers.notes` values, and updates that leave them alone. + +`drivers` has Postgres's default replica identity (a delete carries only the +key) and is partitioned by `status`, so a status change moves a row between +partitions ([`setup.sql`](setup.sql), [`config.yaml`](config.yaml)). + +On a timeline, the workload also runs a 5000-row insert, a bulk delete, a +`TRUNCATE`, adding / dropping / re-adding a column, a bulk update and a status +flip of every driver; and [`run.sh`](run.sh) runs `pg2iceberg compact` twice, +SIGKILLs and restarts pg2iceberg, and runs `pg2iceberg maintain` with a 1m +snapshot retention. + +## What's checked + +- [`compare.py`](compare.py): every row and column of each table, Postgres + against ClickHouse (an independent Iceberg reader), plus ClickHouse's + `count()`. +- `pg2iceberg verify`, in `out/verify.log`. +- The replication slot catches up once the workload stops (to within a few + hundred bytes: the coordinator's own state lives in this Postgres). + +On Colima, bind mounts only work under `$HOME`: run from a checkout there. diff --git a/example/single/compare.py b/example/single/compare.py new file mode 100644 index 0000000..e4e693c --- /dev/null +++ b/example/single/compare.py @@ -0,0 +1,61 @@ +"""Compare Postgres with ClickHouse reading the Iceberg tables pg2iceberg +wrote: every row, every column rendered the same way on both sides, and +ClickHouse's `count()`. Exits non-zero on any difference.""" + +import os +import subprocess +import sys +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +TABLES = ["riders", "drivers", "rides", "payments", "ratings"] + + +def pg(sql): + out = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "postgres", "-d", "rideshare", + "-At", "-F", "\t", "-c", sql], + cwd=HERE, capture_output=True, text=True, check=True) + return out.stdout.splitlines() + + +def ch(sql): + req = urllib.request.Request("http://localhost:8123", data=(sql + " FORMAT TSVRaw").encode()) + return urllib.request.urlopen(req).read().decode().splitlines() + + +def render(name, ty): + """The same text for a value on both sides.""" + if name == "notes": # TOAST-sized: compare digests + return f"coalesce(md5({name}), 'NULL')", f"ifNull(lower(hex(MD5({name}))), 'NULL')" + if ty.startswith("timestamp"): + return (f"coalesce((extract(epoch FROM {name}) * 1000000)::bigint::text, 'NULL')", + f"ifNull(toString(toUnixTimestamp64Micro({name})), 'NULL')") + if ty == "numeric": + return f"coalesce(trim_scale({name})::text, 'NULL')", f"ifNull(toString({name}), 'NULL')" + return f"coalesce({name}::text, 'NULL')", f"ifNull(toString({name}), 'NULL')" + + +ok = True +for table in TABLES: + cols = [tuple(r.split("\t")) for r in pg( + "SELECT column_name, data_type FROM information_schema.columns " + f"WHERE table_schema = 'public' AND table_name = '{table}' ORDER BY ordinal_position")] + pg_exprs, ch_exprs = zip(*(render(n, t) for n, t in cols)) + iceberg = f"rideshare.`rideshare.{table}`" + pg_rows = pg(f"SELECT {', '.join(pg_exprs)} FROM {table}") + ch_rows = ch(f"SELECT {', '.join(ch_exprs)} FROM {iceberg}") + ch_count = int(ch(f"SELECT count() FROM {iceberg}")[0]) + only_pg, only_ch = set(pg_rows) - set(ch_rows), set(ch_rows) - set(pg_rows) + dupes = len(ch_rows) - len(set(ch_rows)) + good = not (only_pg or only_ch or dupes) and ch_count == len(pg_rows) + ok &= good + print(f"{table:9} pg={len(pg_rows):6} iceberg={len(ch_rows):6} count()={ch_count:6} " + f"only_pg={len(only_pg)} only_iceberg={len(only_ch)} duplicates={dupes} " + f"{'OK' if good else 'DIFF'}") + for r in sorted(only_pg)[:3]: + print(" pg :", r[:200]) + for r in sorted(only_ch)[:3]: + print(" iceberg :", r[:200]) +print("ALL MATCH" if ok else "MISMATCHES FOUND") +sys.exit(0 if ok else 1) diff --git a/example/single/config.yaml b/example/single/config.yaml index f685ab0..b00a3a0 100644 --- a/example/single/config.yaml +++ b/example/single/config.yaml @@ -1,20 +1,22 @@ tables: - name: public.riders - name: public.drivers + iceberg: + # A non-key column under DEFAULT replica identity: deletes carry only + # the key, and a status change moves a row between partitions. + partition: ["status"] - name: public.rides - name: public.payments - name: public.ratings source: mode: logical - postgres: host: postgres port: 5432 database: rideshare user: postgres password: postgres - logical: publication_name: pg2iceberg_pub slot_name: pg2iceberg_slot @@ -29,6 +31,5 @@ sink: s3_region: us-east-1 flush_interval: 10s flush_rows: 1000 - flush_bytes: 67108864 state: {} diff --git a/example/single/docker-compose.yml b/example/single/docker-compose.yml index dc050da..f1e99a5 100644 --- a/example/single/docker-compose.yml +++ b/example/single/docker-compose.yml @@ -1,3 +1,6 @@ +# Postgres -> pg2iceberg -> Iceberg REST catalog + MinIO, with ClickHouse to +# query the tables and a mixed workload to drive Postgres. See README.md. +name: pg2iceberg-single services: postgres: image: postgres:16-alpine @@ -5,47 +8,37 @@ services: POSTGRES_DB: rideshare POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - ports: - - "5502:5432" + ports: ["5502:5432"] volumes: - ./schema.sql:/docker-entrypoint-initdb.d/01-schema.sql - ./seed.sql:/docker-entrypoint-initdb.d/02-seed.sql - - pgdata:/var/lib/postgresql/data + - ./setup.sql:/docker-entrypoint-initdb.d/03-setup.sql command: ["postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=4", "-c", "max_wal_senders=4"] healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] + test: ["CMD-SHELL", "pg_isready -U postgres -d rideshare"] interval: 2s timeout: 5s - retries: 10 + retries: 30 minio: - # minio/minio + minio/mc are no longer on Docker Hub; pgsty/* is a - # community-maintained drop-in fork (same binaries and entrypoints). image: pgsty/minio:RELEASE.2026-08-04T00-00-00Z - command: server /data --console-address ":9001" + command: server /data environment: MINIO_ROOT_USER: admin MINIO_ROOT_PASSWORD: password - ports: - - "9000:9000" - - "9001:9001" + ports: ["9000:9000"] healthcheck: test: ["CMD", "mc", "ready", "local"] interval: 2s timeout: 5s - retries: 5 + retries: 15 create-bucket: image: pgsty/mc:RELEASE.2026-09-16T00-00-00Z depends_on: - minio: - condition: service_healthy + minio: {condition: service_healthy} entrypoint: > - /bin/sh -c " - mc alias set myminio http://minio:9000 admin password; - mc mb --ignore-existing myminio/warehouse; - exit 0; - " + /bin/sh -c "mc alias set m http://minio:9000 admin password && mc mb --ignore-existing m/warehouse" iceberg-postgres: image: postgres:16-alpine @@ -57,7 +50,7 @@ services: test: ["CMD-SHELL", "pg_isready -U iceberg"] interval: 2s timeout: 5s - retries: 10 + retries: 15 iceberg-rest: build: ./iceberg-rest @@ -73,50 +66,32 @@ services: CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO CATALOG_S3_ENDPOINT: http://minio:9000 CATALOG_S3_PATH__STYLE__ACCESS: "true" - ports: - - "8181:8181" + ports: ["8181:8181"] depends_on: - iceberg-postgres: - condition: service_healthy - create-bucket: - condition: service_completed_successfully + iceberg-postgres: {condition: service_healthy} + create-bucket: {condition: service_completed_successfully} healthcheck: test: ["CMD-SHELL", "curl -sf http://localhost:8181/v1/config || exit 1"] interval: 3s timeout: 5s - retries: 10 + retries: 30 pg2iceberg: build: context: ../.. dockerfile: Dockerfile - # Dockerfile's ENTRYPOINT is the binary; pass the subcommand here. - # Config is sourced from environment variables (see below); no - # `--config` file needed for this example. command: ["run", "--config", "/etc/pg2iceberg/config.yaml"] + environment: + RUST_LOG: info volumes: - ./config.yaml:/etc/pg2iceberg/config.yaml:ro - environment: - POSTGRES_URL: postgresql://postgres:postgres@postgres:5432/rideshare - TABLES: public.riders,public.drivers,public.rides,public.payments,public.ratings - ICEBERG_CATALOG_URL: http://iceberg-rest:8181 - WAREHOUSE: s3://warehouse/ - NAMESPACE: rideshare - S3_ENDPOINT: http://minio:9000 - S3_ACCESS_KEY: admin - S3_SECRET_KEY: password - ports: - - "9091:9090" depends_on: - postgres: - condition: service_healthy - iceberg-rest: - condition: service_healthy + postgres: {condition: service_healthy} + iceberg-rest: {condition: service_healthy} clickhouse: - image: clickhouse/clickhouse-server:26.2 - ports: - - "8123:8123" + image: clickhouse/clickhouse-server:26.8 + ports: ["8123:8123"] environment: AWS_ACCESS_KEY_ID: admin AWS_SECRET_ACCESS_KEY: password @@ -125,46 +100,16 @@ services: - ./clickhouse/users.xml:/etc/clickhouse-server/users.d/iceberg.xml - ./clickhouse/init.sql:/docker-entrypoint-initdb.d/init.sql depends_on: - iceberg-rest: - condition: service_healthy - - prometheus: - image: prom/prometheus:v3.4.1 - volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml - ports: - - "9090:9090" - depends_on: - pg2iceberg: - condition: service_started + iceberg-rest: {condition: service_healthy} - grafana: - image: grafana/grafana:11.6.0 - environment: - GF_SECURITY_ADMIN_USER: admin - GF_SECURITY_ADMIN_PASSWORD: admin - GF_AUTH_ANONYMOUS_ENABLED: "true" - GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer - volumes: - - ./grafana/provisioning:/etc/grafana/provisioning - - ./grafana/dashboards:/var/lib/grafana/dashboards - ports: - - "3001:3000" - depends_on: - - prometheus - - simulate: + workload: image: python:3.12-slim - depends_on: - postgres: - condition: service_healthy + profiles: ["workload"] environment: DATABASE_URL: postgres://postgres:postgres@postgres:5432/rideshare + DURATION: ${DURATION:-300} volumes: - ./simulate.py:/app/simulate.py - - ../../scripts/requirements.txt:/app/requirements.txt + - ./workload.py:/app/workload.py working_dir: /app - command: sh -c "pip install -q -r requirements.txt && python simulate.py --interval 5" - -volumes: - pgdata: + command: sh -c "pip install -q psycopg2-binary && python -u workload.py" diff --git a/example/single/grafana/dashboards/pg2iceberg.json b/example/single/grafana/dashboards/pg2iceberg.json deleted file mode 120000 index ef4ce80..0000000 --- a/example/single/grafana/dashboards/pg2iceberg.json +++ /dev/null @@ -1 +0,0 @@ -../../../../benchmark/grafana/dashboards/pg2iceberg.json \ No newline at end of file diff --git a/example/single/grafana/provisioning/dashboards/default.yml b/example/single/grafana/provisioning/dashboards/default.yml deleted file mode 100644 index 4825453..0000000 --- a/example/single/grafana/provisioning/dashboards/default.yml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: 1 -providers: - - name: default - orgId: 1 - folder: "" - type: file - disableDeletion: false - editable: true - options: - path: /var/lib/grafana/dashboards - foldersFromFilesStructure: false diff --git a/example/single/grafana/provisioning/datasources/prometheus.yml b/example/single/grafana/provisioning/datasources/prometheus.yml deleted file mode 100644 index 2d43399..0000000 --- a/example/single/grafana/provisioning/datasources/prometheus.yml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: 1 -datasources: - - name: Prometheus - type: prometheus - access: proxy - url: http://prometheus:9090 - isDefault: true - editable: false diff --git a/example/single/prometheus.yml b/example/single/prometheus.yml deleted file mode 100644 index 4f7a62a..0000000 --- a/example/single/prometheus.yml +++ /dev/null @@ -1,10 +0,0 @@ -global: - scrape_interval: 10s - evaluation_interval: 10s - -scrape_configs: - - job_name: pg2iceberg - static_configs: - - targets: ["pg2iceberg:9090"] - labels: - instance: single diff --git a/example/single/run.sh b/example/single/run.sh new file mode 100755 index 0000000..6325f6d --- /dev/null +++ b/example/single/run.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Smoke test: real Postgres -> pg2iceberg -> Iceberg REST + MinIO, under a +# mixed workload (DURATION seconds, default 300) with external compactions, +# a crash and snapshot expiry alongside. Ends with `pg2iceberg verify` and a +# row-by-row comparison of Postgres against ClickHouse reading the Iceberg +# tables. Exits non-zero if they differ. See README.md. +set -u +cd "$(dirname "$0")" +DURATION=${DURATION:-300} +export DURATION +DC="docker compose" +P2I="$DC run --rm -T pg2iceberg" +CFG="--config /etc/pg2iceberg/config.yaml" +mkdir -p out +log() { echo "[$(date +%H:%M:%S)] $*"; } +psql() { $DC exec -T postgres psql -U postgres -d rideshare -At -c "$1"; } + +log "build pg2iceberg" +$DC build -q pg2iceberg || exit 1 + +log "stack up" +$DC up -d postgres minio create-bucket iceberg-postgres iceberg-rest clickhouse +for _ in $(seq 1 90); do + curl -sf http://localhost:8181/v1/config >/dev/null 2>&1 && break; sleep 2 +done +until psql "SELECT 1" >/dev/null 2>&1; do sleep 1; done + +log "pg2iceberg up" +$DC up -d pg2iceberg +sleep 20 + +log "workload start (${DURATION}s)" +$DC --profile workload up -d workload +START=$(date +%s) +at() { while [ $(( $(date +%s) - START )) -lt "$1" ]; do sleep 1; done; } +# Things that happen to pg2iceberg in production, timed into the workload. +step() { + [ "$1" -lt "$DURATION" ] || return 0 + at "$1"; log "$2"; shift 2; "$@"; log " exit $?" +} +step 90 "external compact #1" sh -c "$P2I compact $CFG > out/compact1.log 2>&1" +step 150 "SIGKILL pg2iceberg + restart" sh -c "$DC kill -s SIGKILL pg2iceberg && $DC up -d pg2iceberg" +step 210 "external compact #2" sh -c "$P2I compact $CFG > out/compact2.log 2>&1" +step 240 "maintain (retention 1m)" sh -c "$P2I maintain $CFG --retention 1m > out/maintain.log 2>&1" + +log "waiting for the workload to finish" +while [ -n "$($DC ps -q workload)" ]; do sleep 2; done +$DC --profile workload logs --no-log-prefix workload > out/workload.log 2>&1 +# The coordinator's state lives in this Postgres too, so its own writes +# keep the slot a few hundred bytes behind: wait for it to stop moving. +log "waiting for the slot to catch up" +for _ in $(seq 1 120); do + LAG=$(psql "SELECT pg_wal_lsn_diff(pg_current_wal_lsn(), confirmed_flush_lsn) FROM pg_replication_slots WHERE slot_name = 'pg2iceberg_slot'") + [ "${LAG:-1000000}" -lt 4096 ] && break; sleep 2 +done +log "slot lag: $LAG bytes; letting the materializer drain" +sleep 40 +$DC logs --no-log-prefix pg2iceberg > out/pg2iceberg.log 2>&1 + +log "pg2iceberg verify" +$P2I verify $CFG > out/verify.log 2>&1 +log " exit $?" +log "Postgres vs ClickHouse" +python3 compare.py diff --git a/example/single/setup.sql b/example/single/setup.sql new file mode 100644 index 0000000..290e3ff --- /dev/null +++ b/example/single/setup.sql @@ -0,0 +1,13 @@ +-- Smoke-test extras, after schema + seed. +-- drivers: Postgres's default replica identity (a DELETE/UPDATE's old +-- tuple carries only the key) and a TOAST-able column. +ALTER TABLE drivers REPLICA IDENTITY DEFAULT; +ALTER TABLE drivers ADD COLUMN notes text; +-- More drivers and riders so updates spread out. +INSERT INTO drivers (email, first_name, last_name, phone, city, vehicle_make, vehicle_model, vehicle_year, license_plate, rating, status, signed_up_at) +SELECT 'smoke.driver.' || g || '@example.com', 'D' || g, 'Smoke', '+1-555-9' || g, (ARRAY['San Francisco','New York','Austin'])[1 + g % 3], + 'Toyota', 'Prius', 2024, 'SMK' || g, 4.5, 'active', now() +FROM generate_series(1, 40) g; +INSERT INTO riders (email, first_name, last_name, phone, city, signed_up_at) +SELECT 'smoke.rider.' || g || '@example.com', 'R' || g, 'Smoke', '+1-555-8' || g, (ARRAY['San Francisco','New York','Austin'])[1 + g % 3], now() +FROM generate_series(1, 200) g; diff --git a/example/single/workload.py b/example/single/workload.py new file mode 100644 index 0000000..778c949 --- /dev/null +++ b/example/single/workload.py @@ -0,0 +1,192 @@ +"""Smoke-test workload for pg2iceberg: simulate.py's rideshare actions +(multi-table transactions) plus deletes, churn within a transaction, +rollbacks, key changes, TOAST updates, partition moves, large +transactions, TRUNCATE and schema changes. Two random workers run +concurrently with a timed-events process for DURATION seconds.""" + +import collections +import multiprocessing as mp +import os +import random +import time + +import psycopg2 + +import simulate + +DURATION = float(os.environ.get("DURATION", "300")) +URL = os.environ["DATABASE_URL"] + +# A ~12KB mostly incompressible value: stored out of line (TOAST), so an +# UPDATE that leaves it alone sends pgoutput's "unchanged TOAST" marker. +BIG_NOTE = "(SELECT string_agg(md5(random()::text), '') FROM generate_series(1, 400))" + + +def random_id(cur, table): + cur.execute(f"SELECT id FROM {table} ORDER BY random() LIMIT 1") + row = cur.fetchone() + return row[0] if row else None + + +def delete_rating(conn): + with conn.cursor() as cur: + cur.execute("DELETE FROM ratings WHERE id = (SELECT id FROM ratings ORDER BY random() LIMIT 1)") + conn.commit() + + +def delete_cancelled_ride(conn): + with conn.cursor() as cur: + cur.execute( + "DELETE FROM rides WHERE id = (SELECT r.id FROM rides r WHERE r.status = 'cancelled' " + "AND NOT EXISTS (SELECT 1 FROM payments p WHERE p.ride_id = r.id) " + "AND NOT EXISTS (SELECT 1 FROM ratings g WHERE g.ride_id = r.id) ORDER BY random() LIMIT 1)" + ) + conn.commit() + + +def churn(conn): + """Insert, update and delete one rating, and insert + update another, + in one transaction.""" + with conn.cursor() as cur: + ride = random_id(cur, "rides") + if ride is None: + conn.rollback() + return + cur.execute("INSERT INTO ratings (ride_id, from_rider, score, comment, created_at) " + "VALUES (%s, true, 3, 'churn', now()) RETURNING id", (ride,)) + gone = cur.fetchone()[0] + cur.execute("UPDATE ratings SET score = 4 WHERE id = %s", (gone,)) + cur.execute("DELETE FROM ratings WHERE id = %s", (gone,)) + cur.execute("INSERT INTO ratings (ride_id, from_rider, score, comment, created_at) " + "VALUES (%s, false, 2, 'churn', now()) RETURNING id", (ride,)) + kept = cur.fetchone()[0] + cur.execute("UPDATE ratings SET score = 5, comment = 'churn kept' WHERE id = %s", (kept,)) + conn.commit() + + +def rolled_back(conn): + with conn.cursor() as cur: + ride = random_id(cur, "rides") + if ride is not None: + cur.execute("INSERT INTO ratings (ride_id, from_rider, score, comment, created_at) " + "VALUES (%s, true, 1, 'rolled back', now())", (ride,)) + cur.execute("UPDATE riders SET city = 'Nowhere' WHERE id = (SELECT id FROM riders ORDER BY random() LIMIT 1)") + conn.rollback() + + +def change_key(conn): + with conn.cursor() as cur: + cur.execute("UPDATE ratings SET id = nextval('ratings_id_seq') " + "WHERE id = (SELECT id FROM ratings ORDER BY random() LIMIT 1)") + conn.commit() + + +def toast_write(conn): + with conn.cursor() as cur: + cur.execute(f"UPDATE drivers SET notes = {BIG_NOTE} " + "WHERE id = (SELECT id FROM drivers ORDER BY random() LIMIT 1)") + conn.commit() + + +def toast_untouched(conn): + with conn.cursor() as cur: + cur.execute("UPDATE drivers SET rating = round((4 + random())::numeric, 2) " + "WHERE id = (SELECT id FROM drivers WHERE notes IS NOT NULL ORDER BY random() LIMIT 1)") + conn.commit() + + +def move_partition(conn): + with conn.cursor() as cur: + cur.execute("UPDATE drivers SET status = (ARRAY['active','inactive','suspended'])[1 + floor(random() * 3)::int] " + "WHERE id = (SELECT id FROM drivers ORDER BY random() LIMIT 1)") + conn.commit() + + +def set_tier(conn): + # Fails while the column is dropped; counted as an error. + with conn.cursor() as cur: + cur.execute("UPDATE riders SET tier = (ARRAY['basic','plus','gold'])[1 + floor(random() * 3)::int] " + "WHERE id = (SELECT id FROM riders ORDER BY random() LIMIT 1)") + conn.commit() + + +def simulated(conn): + simulate.pick_action()(conn) + + +MIX = [ + (simulated, 40), (delete_rating, 8), (delete_cancelled_ride, 4), (churn, 8), + (rolled_back, 5), (change_key, 5), (toast_write, 6), (toast_untouched, 10), + (move_partition, 6), (set_tier, 3), +] + + +def worker(seed, deadline, out): + random.seed(seed) + conn = psycopg2.connect(URL) + counts = collections.Counter() + fns, weights = zip(*MIX) + while time.time() < deadline: + fn = random.choices(fns, weights)[0] + try: + fn(conn) + counts[fn.__name__] += 1 + except Exception as e: # noqa: BLE001 — a failed action is just skipped + conn.rollback() + counts[f"error:{fn.__name__}:{type(e).__name__}"] += 1 + time.sleep(random.uniform(0.005, 0.03)) + out.put(dict(counts)) + + +EVENTS = [ + (20, "bulk insert 5000 ratings", + "INSERT INTO ratings (ride_id, from_rider, score, comment, created_at) " + "SELECT (SELECT id FROM rides ORDER BY random() LIMIT 1), true, 5, 'bulk', now() FROM generate_series(1, 5000)"), + (45, "add riders.tier", "ALTER TABLE riders ADD COLUMN tier text DEFAULT 'basic'"), + (70, "bulk delete half the bulk ratings", "DELETE FROM ratings WHERE comment = 'bulk' AND id % 2 = 0"), + (100, "truncate ratings", "TRUNCATE ratings"), + (130, "drop riders.tier", "ALTER TABLE riders DROP COLUMN tier"), + (160, "re-add riders.tier", "ALTER TABLE riders ADD COLUMN tier text"), + (190, "bulk update every ride", "UPDATE rides SET fare_cents = coalesce(fare_cents, 0) + 1"), + (220, "bulk insert 3000 ratings", + "INSERT INTO ratings (ride_id, from_rider, score, comment, created_at) " + "SELECT (SELECT id FROM rides ORDER BY random() LIMIT 1), false, 4, 'bulk2', now() FROM generate_series(1, 3000)"), + (250, "flip every driver's status (partition moves)", + "UPDATE drivers SET status = CASE status WHEN 'active' THEN 'inactive' ELSE 'active' END"), +] + + +def events(start): + conn = psycopg2.connect(URL) + for at, name, sql in EVENTS: + if at >= DURATION: + break + time.sleep(max(0, start + at - time.time())) + t = time.time() + with conn.cursor() as cur: + cur.execute(sql) + n = cur.rowcount + conn.commit() + print(f"[t={at:>3}s] {name}: {n} rows ({time.time() - t:.2f}s)", flush=True) + + +def main(): + start = time.time() + deadline = start + DURATION + out = mp.Queue() + procs = [mp.Process(target=worker, args=(i, deadline, out)) for i in range(2)] + procs.append(mp.Process(target=events, args=(start,))) + for p in procs: + p.start() + total = collections.Counter() + for _ in range(2): + total.update(out.get()) + for p in procs: + p.join() + print("--- workload done ---") + for k, v in sorted(total.items()): + print(f"{k}: {v}") + + +if __name__ == "__main__": + main() diff --git a/tests/cases/00002_update__input.sql b/tests/cases/00002_update__input.sql index ee6c45a..1fe0e45 100644 --- a/tests/cases/00002_update__input.sql +++ b/tests/cases/00002_update__input.sql @@ -11,6 +11,4 @@ INSERT INTO e2e_update (id, name, score) VALUES (2, 'bob', 20), (3, 'charlie', 30); --- NOTE: ClickHouse 26.2 does not apply Iceberg equality deletes yet, --- so the reference output includes both the old and new rows for updated records. UPDATE e2e_update SET score = 99 WHERE id = 2; diff --git a/tests/cases/00003_delete__input.sql b/tests/cases/00003_delete__input.sql index 75c7ae2..8f74559 100644 --- a/tests/cases/00003_delete__input.sql +++ b/tests/cases/00003_delete__input.sql @@ -11,6 +11,4 @@ INSERT INTO e2e_delete (id, name) VALUES (3, 'charlie'), (4, 'diana'); --- NOTE: ClickHouse 26.2 does not apply Iceberg equality deletes yet, --- so the reference output still includes the deleted rows. DELETE FROM e2e_delete WHERE id IN (2, 4); diff --git a/tests/cases/00004_mixed_ops__input.sql b/tests/cases/00004_mixed_ops__input.sql index b12e759..09e8afe 100644 --- a/tests/cases/00004_mixed_ops__input.sql +++ b/tests/cases/00004_mixed_ops__input.sql @@ -13,8 +13,6 @@ INSERT INTO e2e_mixed_ops (id, name, status) VALUES (4, 'diana', 'active'), (5, 'eve', 'active'); --- NOTE: ClickHouse 26.2 does not apply Iceberg equality deletes yet, --- so the reference output includes both old and new rows for updates/deletes. UPDATE e2e_mixed_ops SET status = 'inactive' WHERE id IN (1, 3); DELETE FROM e2e_mixed_ops WHERE id = 5; INSERT INTO e2e_mixed_ops (id, name, status) VALUES (6, 'frank', 'active'); diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index a03e53c..fa691cd 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -84,7 +84,7 @@ services: retries: 10 clickhouse: - image: clickhouse/clickhouse-server:26.2 + image: clickhouse/clickhouse-server:26.8 ports: - "8123:8123" - "9100:9000"