diff --git a/README.md b/README.md index 7f91126..7d5d37a 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ anything running. | `sink-column-flattened` | a sink table names its columns as the source does while a `Flatten` transform on the way delivers the envelope as `after.`, so every column stays at its default | v0.3 | | `mv-column-match` | ClickHouse streaming materialized views match by name, refreshable ones by position. ClickHouse 25.4 and later reject a streaming view that writes a column the target lacks when it is created; a refreshable view's order mismatch was loud on 24.8 and is silent on 26.8 | v0.1 | | `schema-before-connector` | this change adds a column to a captured table and leaves it out of the stream without deciding to, the trap itself, judged on the diff (warning: leaving PII off is right, so it asks for the decision) | v0.2 | -| `replica-identity` | a captured table's replica identity cannot supply what the sink reads | next | +| `replica-identity` | a captured Postgres table has no replica identity (`NOTHING`, or `DEFAULT` with no primary key, after the connector's `replica.identity.autoset.values`): through `pgoutput` Postgres rejects the application's `UPDATE` and `DELETE` on it, through `decoderbufs` Debezium streams only its inserts (warning: an insert-only table loses nothing, and a `cdclint:ignore` says so) | next | | `migration-numbering` | duplicate or gapped migration prefixes | next | Findings name the file, the column, the rule and the fix: diff --git a/corpus/README.md b/corpus/README.md index 39c5d2f..b2a2384 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -32,9 +32,16 @@ The rest exercise one path each: | `ignore-marker-mistakes` | markers used wrong: one with no reason, one naming a rule that does not exist, one covering nothing, each an `ignore-marker` warning; and one on the line above a finding, which acknowledges it | | `golang-migrate-down-files` | Mattermost v10.11.0's `000092_add_createat_to_teammembers.down.sql` drops a column from the wrong table; read in name order it ran just before its up file and deleted `reactions.createat`. Down files are skipped | | `goose-down-section` | a goose file's down section follows its up section; applied whole, every table was created and dropped again. Down sections are skipped | +| `postgres-glued-constraints` | Mattermost v10.11.0 (000006 emoji, 000037 sharedchannelremotes): `UNIQUE(name, deleteat)` and `PRIMARY KEY(id, channelid)` with no space before the parenthesis are table constraints, not columns named after them; the primary key is kept | | `include-table-no-schema` | Stack Overflow 74103659: `table.include.list` names `ipaddrs` without its schema, the connector runs and no topic appears | | `include-table-glob` | Stack Overflow 51345636: `public.bg_*` written as a shell glob; as a regular expression it names no table | | `include-table-typo` | a misspelled `table.include.list` entry, which Debezium only logs as a warning (debezium/dbz#872) | +| `replica-identity-no-primary-key` | a captured table with no primary key on `REPLICA IDENTITY DEFAULT`, read through the default `decoderbufs`: Debezium emits its inserts only | +| `replica-identity-nothing-pgoutput` | a keyed table set to `REPLICA IDENTITY NOTHING`, read through `pgoutput`: Postgres rejects `UPDATE` and `DELETE` on a published table with no replica identity | +| `replica-identity-autoset-nothing` | the connector's `replica.identity.autoset.values` sets `NOTHING` on a keyed table; the finding points at the connector, which is what to edit | +| `replica-identity-autoset-full` | the same keyless table as `replica-identity-no-primary-key`, with the connector setting `FULL`: nothing to report | +| `replica-identity-key-added-later` | the primary key arrives in a later `ALTER TABLE ... ADD CONSTRAINT ... PRIMARY KEY`: nothing to report | +| `replica-identity-key-in-do-block` | Mattermost v10.11.0 `000016_create_reactions.up.sql`, as written: the key is added inside a `DO` block, which cdclint does not run, so it makes no finding rather than guess | | `flatten-bare-columns` | ClickHouse/clickhouse-kafka-connect discussion 182: a `Flatten` transform on the Debezium connector sends `after.`, the ClickHouse table names its columns as Postgres does, and every change lands as a row of zeros | | `flatten-envelope-columns` | ClickHouse's own Postgres CDC example (ClickHouse/examples `cdc/postgresql` at ae417ae): `before.` and `after.` columns resolve to the source columns, so the capture rules see `postcode2`, which the include list added for this entry leaves out. The sink connector config there has no `connector.class`; it is added so the file names its sink | | `flatten-after-unwrap` | a guard: unwrapping first leaves a flat row, so a `Flatten` after it renames nothing | diff --git a/corpus/flatten-bare-columns/migrations/0001_testradiomeas.sql b/corpus/flatten-bare-columns/migrations/0001_testradiomeas.sql index 1d44320..596870f 100644 --- a/corpus/flatten-bare-columns/migrations/0001_testradiomeas.sql +++ b/corpus/flatten-bare-columns/migrations/0001_testradiomeas.sql @@ -9,3 +9,8 @@ CREATE TABLE testradiomeas ( "measNetNoCoverage" INTEGER, "measNetType" TEXT ); + +-- The discussion's sample message carries a whole "before" row (measDate, +-- measLongitude, ...) on a table with no primary key, which only REPLICA +-- IDENTITY FULL produces: with DEFAULT the connector would emit inserts only. +ALTER TABLE testradiomeas REPLICA IDENTITY FULL; diff --git a/corpus/golang-migrate-down-files/migrations/0001_create_tables.up.sql b/corpus/golang-migrate-down-files/migrations/0001_create_tables.up.sql index 99d27dc..44e118d 100644 --- a/corpus/golang-migrate-down-files/migrations/0001_create_tables.up.sql +++ b/corpus/golang-migrate-down-files/migrations/0001_create_tables.up.sql @@ -1,10 +1,20 @@ +-- Reduced from Mattermost v10.11.0 000002 and 000016, keys as Mattermost +-- declares them: teammembers' in its CREATE TABLE, reactions' in a DO block +-- (cdclint does not run DO bodies, so it does not judge reactions' key). CREATE TABLE reactions ( - userid VARCHAR(26) NOT NULL, - postid VARCHAR(26) NOT NULL, - createat BIGINT + userid VARCHAR(26) NOT NULL, + postid VARCHAR(26) NOT NULL, + emojiname VARCHAR(64) NOT NULL, + createat BIGINT ); +DO $$ +BEGIN + ALTER TABLE reactions ADD PRIMARY KEY (postid, userid, emojiname); +END $$; + CREATE TABLE teammembers ( teamid VARCHAR(26) NOT NULL, - userid VARCHAR(26) NOT NULL + userid VARCHAR(26) NOT NULL, + PRIMARY KEY (teamid, userid) ); diff --git a/corpus/postgres-glued-constraints/connector.json b/corpus/postgres-glued-constraints/connector.json new file mode 100644 index 0000000..8603d31 --- /dev/null +++ b/corpus/postgres-glued-constraints/connector.json @@ -0,0 +1,10 @@ +{ + "name": "mattermost-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "plugin.name": "pgoutput", + "topic.prefix": "mm", + "table.include.list": "public.emoji,public.sharedchannelremotes", + "column.include.list": "public.emoji\\.(id|createat|updateat|deleteat|creatorid|name),public.sharedchannelremotes\\.(id|channelid|creatorid|createat|updateat|isinviteaccepted|isinviteconfirmed|remoteid|lastpostupdateat|lastpostid)" + } +} diff --git a/corpus/postgres-glued-constraints/expected.txt b/corpus/postgres-glued-constraints/expected.txt new file mode 100644 index 0000000..cd1f70b --- /dev/null +++ b/corpus/postgres-glued-constraints/expected.txt @@ -0,0 +1 @@ +ok: source, connector and sink agree diff --git a/corpus/postgres-glued-constraints/migrations/000006_create_emojis.up.sql b/corpus/postgres-glued-constraints/migrations/000006_create_emojis.up.sql new file mode 100644 index 0000000..7e07422 --- /dev/null +++ b/corpus/postgres-glued-constraints/migrations/000006_create_emojis.up.sql @@ -0,0 +1,12 @@ +-- Mattermost v10.11.0, server/channels/db/migrations/postgres/ +-- 000006_create_emojis.up.sql, reduced to the table. The table constraint +-- is written UNIQUE(name, deleteat), with no space before the parenthesis. +CREATE TABLE IF NOT EXISTS emoji ( + id VARCHAR(26) PRIMARY KEY, + createat bigint, + updateat bigint, + deleteat bigint, + creatorid VARCHAR(26), + name VARCHAR(64), + UNIQUE(name, deleteat) +); diff --git a/corpus/postgres-glued-constraints/migrations/000037_create_sharedchannelremotes.up.sql b/corpus/postgres-glued-constraints/migrations/000037_create_sharedchannelremotes.up.sql new file mode 100644 index 0000000..dfe2e67 --- /dev/null +++ b/corpus/postgres-glued-constraints/migrations/000037_create_sharedchannelremotes.up.sql @@ -0,0 +1,17 @@ +-- Mattermost v10.11.0, 000037_create_sharedchannelremotes.up.sql, as +-- written: both table constraints are glued to their parenthesis. +CREATE TABLE IF NOT EXISTS sharedchannelremotes ( + id character varying(26) NOT NULL, + channelid character varying(26) NOT NULL, + creatorid character varying(26), + createat bigint, + updateat bigint, + isinviteaccepted boolean, + isinviteconfirmed boolean, + remoteid character varying(26), + PRIMARY KEY(id, channelid), + UNIQUE(channelid, remoteid) +); + +ALTER TABLE sharedchannelremotes ADD COLUMN IF NOT EXISTS LastPostUpdateAt bigint; +ALTER TABLE sharedchannelremotes ADD COLUMN IF NOT EXISTS LastPostId character varying(26); diff --git a/corpus/postgres-glued-constraints/sink/0001_mattermost.sql b/corpus/postgres-glued-constraints/sink/0001_mattermost.sql new file mode 100644 index 0000000..ccac0ec --- /dev/null +++ b/corpus/postgres-glued-constraints/sink/0001_mattermost.sql @@ -0,0 +1,28 @@ +-- Invented sink for the entry: two Kafka-engine tables reading a few of the +-- captured columns. The entry is about the source reader, not the sink. +CREATE TABLE kafka_emoji +( + `id` String, + `name` String, + `deleteat` Int64 +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'mm.public.emoji', + kafka_group_name = 'clickhouse-emoji', + kafka_format = 'JSONEachRow'; + +CREATE TABLE kafka_sharedchannelremotes +( + `id` String, + `channelid` String, + `remoteid` String, + `lastpostupdateat` Int64 +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'mm.public.sharedchannelremotes', + kafka_group_name = 'clickhouse-sharedchannelremotes', + kafka_format = 'JSONEachRow'; diff --git a/corpus/replica-identity-autoset-full/connector.json b/corpus/replica-identity-autoset-full/connector.json new file mode 100644 index 0000000..23d100d --- /dev/null +++ b/corpus/replica-identity-autoset-full/connector.json @@ -0,0 +1,9 @@ +{ + "name": "app-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "topic.prefix": "app", + "table.include.list": "public.audit_events", + "replica.identity.autoset.values": "public.audit_.*:FULL" + } +} diff --git a/corpus/replica-identity-autoset-full/expected.txt b/corpus/replica-identity-autoset-full/expected.txt new file mode 100644 index 0000000..cd1f70b --- /dev/null +++ b/corpus/replica-identity-autoset-full/expected.txt @@ -0,0 +1 @@ +ok: source, connector and sink agree diff --git a/corpus/replica-identity-autoset-full/migrations/0001_audit_events.sql b/corpus/replica-identity-autoset-full/migrations/0001_audit_events.sql new file mode 100644 index 0000000..64298cc --- /dev/null +++ b/corpus/replica-identity-autoset-full/migrations/0001_audit_events.sql @@ -0,0 +1,7 @@ +-- Invented for the entry: a table created without a primary key, as tables +-- meant only to be appended to often are, then captured like the rest. +CREATE TABLE audit_events ( + actor_id UUID NOT NULL, + action TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/corpus/replica-identity-autoset-full/sink/0001_audit_events.sql b/corpus/replica-identity-autoset-full/sink/0001_audit_events.sql new file mode 100644 index 0000000..13b77f4 --- /dev/null +++ b/corpus/replica-identity-autoset-full/sink/0001_audit_events.sql @@ -0,0 +1,14 @@ +-- Invented sink for the entry: a Kafka-engine table reading the captured +-- table. The entry is about the source table's replica identity. +CREATE TABLE kafka_audit_events +( + `actor_id` String, + `action` String, + `created_at` String +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'app.public.audit_events', + kafka_group_name = 'clickhouse-audit_events', + kafka_format = 'JSONEachRow'; diff --git a/corpus/replica-identity-autoset-nothing/connector.json b/corpus/replica-identity-autoset-nothing/connector.json new file mode 100644 index 0000000..d26094b --- /dev/null +++ b/corpus/replica-identity-autoset-nothing/connector.json @@ -0,0 +1,9 @@ +{ + "name": "app-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "topic.prefix": "app", + "table.include.list": "public.orders", + "replica.identity.autoset.values": "public.orders:NOTHING" + } +} diff --git a/corpus/replica-identity-autoset-nothing/expected.txt b/corpus/replica-identity-autoset-nothing/expected.txt new file mode 100644 index 0000000..9564c18 --- /dev/null +++ b/corpus/replica-identity-autoset-nothing/expected.txt @@ -0,0 +1,5 @@ +warning replica-identity replica-identity-autoset-nothing/connector.json + public.orders is captured with REPLICA IDENTITY NOTHING, set by replica.identity.autoset.values in replica-identity-autoset-nothing/connector.json, so it has no replica identity + Debezium's update and delete events for it carry no previous values, so a delete has no before to say which row went + fix: take public.orders out of replica.identity.autoset.values, or set it to DEFAULT (it has a primary key) or FULL; if it only ever takes inserts, say so on the line that creates it: -- cdclint:ignore replica-identity: append-only +0 error(s), 1 warning(s), 0 info diff --git a/corpus/replica-identity-autoset-nothing/migrations/0001_orders.sql b/corpus/replica-identity-autoset-nothing/migrations/0001_orders.sql new file mode 100644 index 0000000..9c16211 --- /dev/null +++ b/corpus/replica-identity-autoset-nothing/migrations/0001_orders.sql @@ -0,0 +1,6 @@ +-- Invented for the entry: a keyed table whose replica identity the +-- connector overrides with replica.identity.autoset.values. +CREATE TABLE orders ( + id UUID PRIMARY KEY, + status TEXT NOT NULL +); diff --git a/corpus/replica-identity-autoset-nothing/sink/0001_orders.sql b/corpus/replica-identity-autoset-nothing/sink/0001_orders.sql new file mode 100644 index 0000000..c6b7395 --- /dev/null +++ b/corpus/replica-identity-autoset-nothing/sink/0001_orders.sql @@ -0,0 +1,13 @@ +-- Invented sink for the entry: a Kafka-engine table reading the captured +-- table. The entry is about the source table's replica identity. +CREATE TABLE kafka_orders +( + `id` String, + `status` String +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'app.public.orders', + kafka_group_name = 'clickhouse-orders', + kafka_format = 'JSONEachRow'; diff --git a/corpus/replica-identity-key-added-later/connector.json b/corpus/replica-identity-key-added-later/connector.json new file mode 100644 index 0000000..0f2574d --- /dev/null +++ b/corpus/replica-identity-key-added-later/connector.json @@ -0,0 +1,8 @@ +{ + "name": "app-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "topic.prefix": "app", + "table.include.list": "public.invoices" + } +} diff --git a/corpus/replica-identity-key-added-later/expected.txt b/corpus/replica-identity-key-added-later/expected.txt new file mode 100644 index 0000000..cd1f70b --- /dev/null +++ b/corpus/replica-identity-key-added-later/expected.txt @@ -0,0 +1 @@ +ok: source, connector and sink agree diff --git a/corpus/replica-identity-key-added-later/migrations/0001_invoices.sql b/corpus/replica-identity-key-added-later/migrations/0001_invoices.sql new file mode 100644 index 0000000..4073063 --- /dev/null +++ b/corpus/replica-identity-key-added-later/migrations/0001_invoices.sql @@ -0,0 +1,7 @@ +-- Invented for the entry: the primary key is added after the table, by +-- ALTER TABLE, as migrations that split constraints out often do. +CREATE TABLE invoices ( + id UUID NOT NULL, + total NUMERIC NOT NULL +); +ALTER TABLE invoices ADD CONSTRAINT invoices_pkey PRIMARY KEY (id); diff --git a/corpus/replica-identity-key-added-later/sink/0001_invoices.sql b/corpus/replica-identity-key-added-later/sink/0001_invoices.sql new file mode 100644 index 0000000..023d83a --- /dev/null +++ b/corpus/replica-identity-key-added-later/sink/0001_invoices.sql @@ -0,0 +1,13 @@ +-- Invented sink for the entry: a Kafka-engine table reading the captured +-- table. The entry is about the source table's replica identity. +CREATE TABLE kafka_invoices +( + `id` String, + `total` String +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'app.public.invoices', + kafka_group_name = 'clickhouse-invoices', + kafka_format = 'JSONEachRow'; diff --git a/corpus/replica-identity-key-in-do-block/connector.json b/corpus/replica-identity-key-in-do-block/connector.json new file mode 100644 index 0000000..7e75db5 --- /dev/null +++ b/corpus/replica-identity-key-in-do-block/connector.json @@ -0,0 +1,8 @@ +{ + "name": "mm-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "topic.prefix": "mm", + "table.include.list": "public.reactions" + } +} diff --git a/corpus/replica-identity-key-in-do-block/expected.txt b/corpus/replica-identity-key-in-do-block/expected.txt new file mode 100644 index 0000000..cd1f70b --- /dev/null +++ b/corpus/replica-identity-key-in-do-block/expected.txt @@ -0,0 +1 @@ +ok: source, connector and sink agree diff --git a/corpus/replica-identity-key-in-do-block/migrations/000016_create_reactions.up.sql b/corpus/replica-identity-key-in-do-block/migrations/000016_create_reactions.up.sql new file mode 100644 index 0000000..b4d320f --- /dev/null +++ b/corpus/replica-identity-key-in-do-block/migrations/000016_create_reactions.up.sql @@ -0,0 +1,35 @@ +-- Mattermost v10.11.0, server/channels/db/migrations/postgres/ +-- 000016_create_reactions.up.sql, as written: the table is created with no +-- primary key and the DO block below adds one. cdclint does not run DO +-- bodies, so it cannot know the key, and makes no replica-identity finding. +CREATE TABLE IF NOT EXISTS reactions( + userid VARCHAR(26) NOT NULL, + postid VARCHAR(26) NOT NULL, + emojiname VARCHAR(64) NOT NULL, + createat bigint +); + +ALTER TABLE reactions ADD COLUMN IF NOT EXISTS updateat bigint; +ALTER TABLE reactions ADD COLUMN IF NOT EXISTS deleteat bigint; + +DO $$ +<> +DECLARE + existing_index text; +BEGIN + SELECT string_agg(a.attname, ',') INTO existing_index + FROM pg_constraint AS c + CROSS JOIN + (SELECT unnest(conkey) FROM pg_constraint WHERE conrelid = 'reactions'::regclass AND contype='p') AS cols(colnum) + INNER JOIN pg_attribute AS a ON a.attrelid = c.conrelid AND cols.colnum = a.attnum + WHERE c.contype = 'p' + AND c.conrelid = 'reactions'::regclass; + + IF COALESCE (existing_index, '') <> text('postid,userid,emojiname') THEN + ALTER TABLE reactions + DROP CONSTRAINT IF EXISTS reactions_pkey, + ADD PRIMARY KEY (postid, userid, emojiname); + END IF; +END alter_pk $$; + +ALTER TABLE reactions ADD COLUMN IF NOT EXISTS remoteid VARCHAR(26); diff --git a/corpus/replica-identity-key-in-do-block/sink/0001_reactions.sql b/corpus/replica-identity-key-in-do-block/sink/0001_reactions.sql new file mode 100644 index 0000000..1b8f1f8 --- /dev/null +++ b/corpus/replica-identity-key-in-do-block/sink/0001_reactions.sql @@ -0,0 +1,14 @@ +-- Invented sink for the entry: a Kafka-engine table reading the captured +-- table. The entry is about the source table's replica identity. +CREATE TABLE kafka_reactions +( + `userid` String, + `postid` String, + `emojiname` String +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'mm.public.reactions', + kafka_group_name = 'clickhouse-reactions', + kafka_format = 'JSONEachRow'; diff --git a/corpus/replica-identity-no-primary-key/connector.json b/corpus/replica-identity-no-primary-key/connector.json new file mode 100644 index 0000000..1799bcd --- /dev/null +++ b/corpus/replica-identity-no-primary-key/connector.json @@ -0,0 +1,8 @@ +{ + "name": "app-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "topic.prefix": "app", + "table.include.list": "public.audit_events" + } +} diff --git a/corpus/replica-identity-no-primary-key/expected.txt b/corpus/replica-identity-no-primary-key/expected.txt new file mode 100644 index 0000000..f3c4cc6 --- /dev/null +++ b/corpus/replica-identity-no-primary-key/expected.txt @@ -0,0 +1,5 @@ +warning replica-identity replica-identity-no-primary-key/migrations/0001_audit_events.sql:3 + public.audit_events is captured with REPLICA IDENTITY DEFAULT and no primary key, so it has no replica identity + Debezium emits only its inserts: updates and deletes never reach the stream, and the sink keeps every row as it was first written + fix: ALTER TABLE public.audit_events REPLICA IDENTITY FULL; in a new migration, or give it a primary key; if it only ever takes inserts, say so on the line that creates it: -- cdclint:ignore replica-identity: append-only +0 error(s), 1 warning(s), 0 info diff --git a/corpus/replica-identity-no-primary-key/migrations/0001_audit_events.sql b/corpus/replica-identity-no-primary-key/migrations/0001_audit_events.sql new file mode 100644 index 0000000..64298cc --- /dev/null +++ b/corpus/replica-identity-no-primary-key/migrations/0001_audit_events.sql @@ -0,0 +1,7 @@ +-- Invented for the entry: a table created without a primary key, as tables +-- meant only to be appended to often are, then captured like the rest. +CREATE TABLE audit_events ( + actor_id UUID NOT NULL, + action TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/corpus/replica-identity-no-primary-key/sink/0001_audit_events.sql b/corpus/replica-identity-no-primary-key/sink/0001_audit_events.sql new file mode 100644 index 0000000..13b77f4 --- /dev/null +++ b/corpus/replica-identity-no-primary-key/sink/0001_audit_events.sql @@ -0,0 +1,14 @@ +-- Invented sink for the entry: a Kafka-engine table reading the captured +-- table. The entry is about the source table's replica identity. +CREATE TABLE kafka_audit_events +( + `actor_id` String, + `action` String, + `created_at` String +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'app.public.audit_events', + kafka_group_name = 'clickhouse-audit_events', + kafka_format = 'JSONEachRow'; diff --git a/corpus/replica-identity-nothing-pgoutput/connector.json b/corpus/replica-identity-nothing-pgoutput/connector.json new file mode 100644 index 0000000..22730f5 --- /dev/null +++ b/corpus/replica-identity-nothing-pgoutput/connector.json @@ -0,0 +1,9 @@ +{ + "name": "app-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "topic.prefix": "app", + "table.include.list": "public.sessions", + "plugin.name": "pgoutput" + } +} diff --git a/corpus/replica-identity-nothing-pgoutput/expected.txt b/corpus/replica-identity-nothing-pgoutput/expected.txt new file mode 100644 index 0000000..3a1ad64 --- /dev/null +++ b/corpus/replica-identity-nothing-pgoutput/expected.txt @@ -0,0 +1,5 @@ +warning replica-identity replica-identity-nothing-pgoutput/migrations/0001_sessions.sql:3 + public.sessions is captured with REPLICA IDENTITY NOTHING, so it has no replica identity + Postgres rejects UPDATE and DELETE on a table with no replica identity while a publication replicates them, and this connector reads through one (plugin.name pgoutput): the application's own writes to it fail + fix: ALTER TABLE public.sessions REPLICA IDENTITY DEFAULT; in a new migration (it has a primary key), or FULL; if it only ever takes inserts, say so on the line that creates it: -- cdclint:ignore replica-identity: append-only +0 error(s), 1 warning(s), 0 info diff --git a/corpus/replica-identity-nothing-pgoutput/migrations/0001_sessions.sql b/corpus/replica-identity-nothing-pgoutput/migrations/0001_sessions.sql new file mode 100644 index 0000000..1d470d4 --- /dev/null +++ b/corpus/replica-identity-nothing-pgoutput/migrations/0001_sessions.sql @@ -0,0 +1,8 @@ +-- Invented for the entry: a keyed table later set to REPLICA IDENTITY +-- NOTHING, read through pgoutput, so through a publication. +CREATE TABLE sessions ( + id UUID PRIMARY KEY, + user_id UUID NOT NULL, + expires_at TIMESTAMPTZ NOT NULL +); +ALTER TABLE sessions REPLICA IDENTITY NOTHING; diff --git a/corpus/replica-identity-nothing-pgoutput/sink/0001_sessions.sql b/corpus/replica-identity-nothing-pgoutput/sink/0001_sessions.sql new file mode 100644 index 0000000..b34d2ed --- /dev/null +++ b/corpus/replica-identity-nothing-pgoutput/sink/0001_sessions.sql @@ -0,0 +1,14 @@ +-- Invented sink for the entry: a Kafka-engine table reading the captured +-- table. The entry is about the source table's replica identity. +CREATE TABLE kafka_sessions +( + `id` String, + `user_id` String, + `expires_at` String +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'app.public.sessions', + kafka_group_name = 'clickhouse-sessions', + kafka_format = 'JSONEachRow'; diff --git a/internal/capture/debezium/debezium.go b/internal/capture/debezium/debezium.go index ab9e0b8..7fea13d 100644 --- a/internal/capture/debezium/debezium.go +++ b/internal/capture/debezium/debezium.go @@ -40,12 +40,22 @@ type Contract struct { columnPatterns []string tablePatterns []string routers []router + autoset []autoset + autosetErr bool shape model.Shape Config map[string]string Class string Name string } +// autoset is one pair of replica.identity.autoset.values: a regular +// expression over schema.table and the identity the connector sets on the +// tables it captures that match it. +type autoset struct { + re *regexp.Regexp + identity model.ReplicaIdentity +} + type router struct { re *regexp.Regexp replacement string @@ -128,6 +138,7 @@ func Parse(b []byte) (*Contract, error) { if c.columnListKey == "" { c.columnListKey = "column.include.list" } + c.autoset, c.autosetErr = autosets(cfg["replica.identity.autoset.values"]) if c.tableListKey == "" { c.tableListKey = "table.include.list" } @@ -333,3 +344,71 @@ func (c *Contract) ColumnPatterns() []string { return c.columnPatterns } // reason as columns. func (c *Contract) TablePatterns() []string { return c.tablePatterns } func (c *Contract) TableListSetting() string { return c.tableListKey } + +// autosets reads replica.identity.autoset.values, documented as +// comma-separated ":" pairs, identity one of DEFAULT, +// FULL, NOTHING or INDEX . A pair it cannot read makes the whole +// setting unreadable (bad), so the replica-identity rule makes no finding +// rather than judge a table by a setting it misread. +func autosets(v string) (out []autoset, bad bool) { + for _, pair := range strings.Split(v, ",") { + pair = strings.TrimSpace(pair) + if pair == "" { + continue + } + i := strings.LastIndex(pair, ":") + if i <= 0 { + return nil, true + } + // Matched like the include lists: whole name, ignoring case. + re, err := regexp.Compile("(?i)^(?:" + strings.TrimSpace(pair[:i]) + ")$") + if err != nil { + return nil, true + } + var id model.ReplicaIdentity + switch value := strings.ToUpper(strings.TrimSpace(pair[i+1:])); { + case value == "DEFAULT": + id = model.ReplicaDefault + case value == "FULL": + id = model.ReplicaFull + case value == "NOTHING": + id = model.ReplicaNothing + case strings.HasPrefix(value, "INDEX "): + id = model.ReplicaIndex + default: + return nil, true + } + out = append(out, autoset{re, id}) + } + return out, false +} + +// ReplicaIdentityOverride is the identity replica.identity.autoset.values +// gives a captured table. set is false when no pair names the table. ok is +// false when the setting cannot be read or two pairs give the table +// different identities: the documentation does not say which wins. +func (c *Contract) ReplicaIdentityOverride(schema, table string) (id model.ReplicaIdentity, set, ok bool) { + if c.autosetErr { + return "", true, false + } + q := schema + "." + table + for _, a := range c.autoset { + if !a.re.MatchString(q) { + continue + } + if set && a.identity != id { + return "", true, false + } + id, set = a.identity, true + } + return id, set, true +} + +// Plugin is plugin.name, the logical decoding plug-in; decoderbufs when +// unset, Debezium's documented default. +func (c *Contract) Plugin() string { + if p := strings.TrimSpace(c.Config["plugin.name"]); p != "" { + return strings.ToLower(p) + } + return "decoderbufs" +} diff --git a/internal/capture/debezium/debezium_test.go b/internal/capture/debezium/debezium_test.go index 6d86232..4962c95 100644 --- a/internal/capture/debezium/debezium_test.go +++ b/internal/capture/debezium/debezium_test.go @@ -1,6 +1,10 @@ package debezium -import "testing" +import ( + "testing" + + "github.com/avison9/cdclint/internal/model" +) func TestIncludeListsAndTopics(t *testing.T) { c, err := Parse([]byte(`{"name":"pg","config":{ @@ -55,3 +59,39 @@ func TestExcludeList(t *testing.T) { t.Errorf("column list key = %q", c.ColumnListSetting()) } } + +func TestReplicaIdentityAutosetValues(t *testing.T) { + // The documented form: comma-separated : pairs. + c, err := Parse([]byte(`{"config":{"replica.identity.autoset.values":"public.audit_.*:FULL, public.orders:NOTHING, s2.t3:INDEX idx_name"}}`)) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + schema, table string + id model.ReplicaIdentity + set bool + }{ + {"public", "audit_events", model.ReplicaFull, true}, + {"public", "ORDERS", model.ReplicaNothing, true}, + {"s2", "t3", model.ReplicaIndex, true}, + {"public", "users", "", false}, + } { + id, set, ok := c.ReplicaIdentityOverride(tc.schema, tc.table) + if !ok || set != tc.set || id != tc.id { + t.Errorf("%s.%s: id %q set %v ok %v, want %q %v true", tc.schema, tc.table, id, set, ok, tc.id, tc.set) + } + } + if c.Plugin() != "decoderbufs" { + t.Errorf("plugin = %q, want the documented default decoderbufs", c.Plugin()) + } + // Two pairs that disagree about one table, and a pair that cannot be + // read, both leave the identity unknown rather than guessed. + two, _ := Parse([]byte(`{"config":{"replica.identity.autoset.values":"public.*:FULL,public.orders:NOTHING"}}`)) + if _, set, ok := two.ReplicaIdentityOverride("public", "orders"); !set || ok { + t.Errorf("conflicting pairs: set %v ok %v, want set and not ok", set, ok) + } + bad, _ := Parse([]byte(`{"config":{"replica.identity.autoset.values":"public.orders:EVERYTHING"}}`)) + if _, _, ok := bad.ReplicaIdentityOverride("public", "users"); ok { + t.Error("an unreadable setting should not be trusted for any table") + } +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go index c89acd5..02dc32f 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -199,6 +199,7 @@ func Run(in *Input) []model.Finding { fs = append(fs, sourceColumnNotCaptured(in, reads, raised)...) fs = append(fs, capturedColumnMissing(in)...) fs = append(fs, capturedTableMissing(in)...) + fs = append(fs, replicaIdentity(in)...) fs = append(fs, mvColumnMatch(in)...) model.Sort(fs) return fs diff --git a/internal/engine/replica.go b/internal/engine/replica.go new file mode 100644 index 0000000..96f7920 --- /dev/null +++ b/internal/engine/replica.go @@ -0,0 +1,96 @@ +package engine + +import ( + "fmt" + + "github.com/avison9/cdclint/internal/model" +) + +// ReplicaIdentityContract is implemented by contracts that can override a +// table's replica identity (Debezium's replica.identity.autoset.values) +// and name the logical decoding plug-in they read through. +type ReplicaIdentityContract interface { + ReplicaIdentityOverride(schema, table string) (id model.ReplicaIdentity, set, ok bool) + Plugin() string +} + +// replicaIdentity raises a captured Postgres table with no replica +// identity: REPLICA IDENTITY NOTHING, or DEFAULT with no primary key. +// Both consequences are documented, not inferred. Postgres: "Tables with a +// replica identity defined as NOTHING, DEFAULT without a primary key, or +// USING INDEX with a dropped index, cannot support UPDATE or DELETE +// operations when included in a publication replicating these actions" +// (logical-replication-publication), which is the pgoutput case. Debezium +// (PostgreSQL connector, "Replica identity"): with DEFAULT and no primary +// key "the connector does not emit UPDATE or DELETE events for that +// table", and with NOTHING its update and delete events "do not contain +// any information about the previous value of any table column". +// +// USING INDEX is not judged: whether the index still exists is not in the +// files. Neither is a table whose key may be set where the reader cannot +// see (KeyUnknown), nor one whose connector override cannot be read. A +// warning, not an error: a table that only ever takes inserts loses +// nothing, and the fix says how to record that. +func replicaIdentity(in *Input) []model.Finding { + rc, _ := in.Contract.(ReplicaIdentityContract) + var fs []model.Finding + for _, t := range in.Source.Tables { + // MySQL tables carry no replica identity; the setting is Postgres'. + if t.ReplicaIdentity == "" || t.KeyUnknown || !in.Contract.CapturesTable(t.Schema, t.Name) { + continue + } + id, pos, overridden := t.ReplicaIdentity, t.Pos, false + if rc != nil { + o, set, ok := rc.ReplicaIdentityOverride(t.Schema, t.Name) + if !ok { + continue + } + if set { + // The connector sets it on every start, so the + // override is what the stream sees, and the + // connector file is what to edit. + id, pos, overridden = o, in.Contract.Pos(), true + } + } + keyed := len(t.PrimaryKey) > 0 + var what string + switch { + case id == model.ReplicaNothing: + what = "REPLICA IDENTITY NOTHING" + case id == model.ReplicaDefault && !keyed: + what = "REPLICA IDENTITY DEFAULT and no primary key" + default: + continue + } + q := t.Qualified() + if overridden { + what += fmt.Sprintf(", set by replica.identity.autoset.values in %s", in.Contract.Pos().File) + } + var consequence string + switch { + case rc != nil && rc.Plugin() == "pgoutput": + consequence = "Postgres rejects UPDATE and DELETE on a table with no replica identity while a publication replicates them, and this connector reads through one (plugin.name pgoutput): the application's own writes to it fail" + case id == model.ReplicaNothing: + consequence = "Debezium's update and delete events for it carry no previous values, so a delete has no before to say which row went" + default: + consequence = "Debezium emits only its inserts: updates and deletes never reach the stream, and the sink keeps every row as it was first written" + } + var fix string + switch { + case overridden && keyed: + fix = fmt.Sprintf("take %s out of replica.identity.autoset.values, or set it to DEFAULT (it has a primary key) or FULL", q) + case overridden: + fix = fmt.Sprintf("set %s to FULL in replica.identity.autoset.values, or give it a primary key", q) + case keyed: + fix = fmt.Sprintf("ALTER TABLE %s REPLICA IDENTITY DEFAULT; in a new migration (it has a primary key), or FULL", q) + default: + fix = fmt.Sprintf("ALTER TABLE %s REPLICA IDENTITY FULL; in a new migration, or give it a primary key", q) + } + fs = append(fs, model.Finding{ + Rule: "replica-identity", Severity: model.Warning, Pos: pos, + Message: fmt.Sprintf("%s is captured with %s, so it has no replica identity\n%s", q, what, consequence), + Fix: fix + "; if it only ever takes inserts, say so on the line that creates it: -- cdclint:ignore replica-identity: append-only", + }) + } + return fs +} diff --git a/internal/engine/rules.go b/internal/engine/rules.go index 06ed784..3d39e7b 100644 --- a/internal/engine/rules.go +++ b/internal/engine/rules.go @@ -11,6 +11,7 @@ var Rules = []string{ "source-column-not-captured", "captured-column-missing", "captured-table-missing", + "replica-identity", "topic-table-mapping", "sink-column-flattened", "mv-column-match", diff --git a/internal/model/model.go b/internal/model/model.go index df8cbb3..48fadb7 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -54,7 +54,12 @@ type Table struct { Columns []Column PrimaryKey []string ReplicaIdentity ReplicaIdentity - Pos Pos + // KeyUnknown is set when a statement the reader does not run may have + // given the table its primary key or replica identity: a DO block that + // alters it (Mattermost adds reactions' key that way), or PRIMARY KEY + // USING INDEX. Rules that depend on the key make no finding then. + KeyUnknown bool + Pos Pos } // Qualified is schema.name, the form Debezium matches include lists against. diff --git a/internal/source/postgres/postgres.go b/internal/source/postgres/postgres.go index a21e83e..64d7aa2 100644 --- a/internal/source/postgres/postgres.go +++ b/internal/source/postgres/postgres.go @@ -10,6 +10,7 @@ package postgres import ( "fmt" + "regexp" "strings" "github.com/avison9/cdclint/internal/ddl" @@ -64,11 +65,30 @@ func Apply(src *model.Source, file, text string) error { alterTable(src, st.Text, w, pos) case ddl.HasPrefixFold(w, "DROP", "TABLE"): dropTable(src, w) + case ddl.HasPrefixFold(w, "DO"): + doBlock(src, st.Text) } } return nil } +// alterInDo finds the tables a DO block's body alters. +var alterInDo = regexp.MustCompile(`(?i)\bALTER\s+TABLE\s+(?:ONLY\s+)?(?:IF\s+EXISTS\s+)?((?:"[^"]+"|[\w$]+)(?:\.(?:"[^"]+"|[\w$]+))?)`) + +// doBlock does not run a DO block: its body is procedural, often guarded +// by a condition on the catalog. It marks the tables the body alters as +// having an unknown key, so a rule that depends on the key stays quiet +// rather than judge a table whose key may be added there (Mattermost +// 000016_create_reactions.up.sql adds reactions' primary key in one). +func doBlock(src *model.Source, text string) { + for _, m := range alterInDo.FindAllStringSubmatch(text, -1) { + schema, table := splitQualified(m[1]) + if t := src.Table(schema, table); t != nil { + t.KeyUnknown = true + } + } +} + func splitQualified(name string) (schema, table string) { parts := ddl.SplitName(name) if len(parts) >= 2 { @@ -117,9 +137,9 @@ func createTable(src *model.Source, text string, pos model.Pos) { if len(iw) == 0 { continue } - switch strings.ToUpper(iw[0]) { + switch keyword(iw[0]) { case "CONSTRAINT", "PRIMARY", "UNIQUE", "CHECK", "FOREIGN", "EXCLUDE", "LIKE": - if ddl.HasPrefixFold(iw, "PRIMARY", "KEY") { + if hasKeywords(iw, "PRIMARY", "KEY") { t.PrimaryKey = keyColumns(item) } if ddl.HasPrefixFold(iw, "CONSTRAINT") && containsFold(iw, "PRIMARY") { @@ -145,7 +165,7 @@ func createTable(src *model.Source, text string, pos model.Pos) { // the first constraint keyword. func typeWords(w []string) []string { for i, x := range w { - switch strings.ToUpper(x) { + switch keyword(x) { case "NOT", "NULL", "DEFAULT", "PRIMARY", "REFERENCES", "UNIQUE", "CHECK", "GENERATED", "CONSTRAINT", "COLLATE": return w[:i] } @@ -165,6 +185,34 @@ func keyColumns(item string) []string { return cols } +// keyword is a word as a keyword: upper case, and cut at an opening +// parenthesis. ddl.Words keeps a parenthesised group glued to the word +// before it, and Postgres needs no space there: Mattermost writes +// UNIQUE(channelid, remoteid) and PRIMARY KEY(id, channelid), which were +// read as a column named "UNIQUE(channelid, remoteid)" and as no primary +// key. Kept to this reader: in the ClickHouse reader a word such as +// Kafka(...) is compared whole on purpose. +func keyword(w string) string { + if i := strings.IndexByte(w, '('); i >= 0 { + w = w[:i] + } + return strings.ToUpper(w) +} + +// hasKeywords is ddl.HasPrefixFold for keywords that may be glued to a +// parenthesis. +func hasKeywords(w []string, kws ...string) bool { + if len(w) < len(kws) { + return false + } + for i, kw := range kws { + if keyword(w[i]) != kw { + return false + } + } + return true +} + func containsFold(w []string, kw string) bool { for _, x := range w { if strings.EqualFold(x, kw) { @@ -199,7 +247,7 @@ func alterTable(src *model.Source, text string, w []string, pos model.Pos) { continue } switch { - case ddl.HasPrefixFold(aw, "ADD", "COLUMN"), ddl.HasPrefixFold(aw, "ADD") && !ddl.HasPrefixFold(aw, "ADD", "CONSTRAINT") && !ddl.HasPrefixFold(aw, "ADD", "PRIMARY") && !ddl.HasPrefixFold(aw, "ADD", "UNIQUE") && !ddl.HasPrefixFold(aw, "ADD", "CHECK") && !ddl.HasPrefixFold(aw, "ADD", "FOREIGN") && !ddl.HasPrefixFold(aw, "ADD", "EXCLUDE"): + case ddl.HasPrefixFold(aw, "ADD", "COLUMN"), ddl.HasPrefixFold(aw, "ADD") && !hasKeywords(aw, "ADD", "CONSTRAINT") && !hasKeywords(aw, "ADD", "PRIMARY") && !hasKeywords(aw, "ADD", "UNIQUE") && !hasKeywords(aw, "ADD", "CHECK") && !hasKeywords(aw, "ADD", "FOREIGN") && !hasKeywords(aw, "ADD", "EXCLUDE"): j := 1 if strings.EqualFold(aw[j], "COLUMN") { j++ @@ -240,6 +288,14 @@ func alterTable(src *model.Source, text string, w []string, pos model.Pos) { c.Pos = pos } } + case hasKeywords(aw, "ADD", "PRIMARY", "KEY"), hasKeywords(aw, "ADD", "CONSTRAINT") && len(aw) > 3 && hasKeywords(aw[3:], "PRIMARY", "KEY"): + // A key added after the table, as migrations that split + // constraints out do. USING INDEX names no columns here. + if cols := keyColumns(a); len(cols) > 0 { + t.PrimaryKey = cols + } else { + t.KeyUnknown = true + } case ddl.HasPrefixFold(aw, "RENAME", "TO"): if len(aw) >= 3 { t.Name = ddl.Unquote(aw[2]) diff --git a/internal/source/postgres/postgres_test.go b/internal/source/postgres/postgres_test.go index dd2994b..7bbf528 100644 --- a/internal/source/postgres/postgres_test.go +++ b/internal/source/postgres/postgres_test.go @@ -103,3 +103,76 @@ func TestColumnsAnAlterAddsAreOnTheirOwnLines(t *testing.T) { } } } + +func TestAConstraintGluedToItsParenthesisIsNotAColumn(t *testing.T) { + // Mattermost 000037_create_sharedchannelremotes.up.sql writes + // PRIMARY KEY(id, channelid) and UNIQUE(channelid, remoteid) with no + // space: the word splitter keeps "UNIQUE(channelid, remoteid)" as one + // word, which was read as a column of that name, and "KEY(id, + // channelid)", which hid the primary key. + src := &model.Source{} + text := `CREATE TABLE IF NOT EXISTS sharedchannelremotes ( + id character varying(26) NOT NULL, + channelid character varying(26) NOT NULL, + remoteid character varying(26), + price int CHECK(price > 0), + PRIMARY KEY(id, channelid), + UNIQUE(channelid, remoteid), + CHECK(remoteid <> '') +); +ALTER TABLE sharedchannelremotes ADD UNIQUE(remoteid); +ALTER TABLE sharedchannelremotes ADD CHECK(price < 100); +ALTER TABLE sharedchannelremotes ADD COLUMN note TEXT;` + if err := Apply(src, "000037.up.sql", text); err != nil { + t.Fatal(err) + } + r := src.Table("public", "sharedchannelremotes") + if r == nil { + t.Fatal("table missing") + } + var names []string + for _, c := range r.Columns { + names = append(names, c.Name+":"+c.Type) + } + want := "[id:character varying(26) channelid:character varying(26) remoteid:character varying(26) price:int note:TEXT]" + if got := formatSlice(names); got != want { + t.Errorf("columns = %s, want %s", got, want) + } + if formatSlice(r.PrimaryKey) != "[id channelid]" { + t.Errorf("primary key = %v, want [id channelid]", r.PrimaryKey) + } +} + +func TestAKeyAddedLaterOrInADoBlock(t *testing.T) { + src := &model.Source{} + text := `CREATE TABLE invoices (id UUID NOT NULL, total NUMERIC); +ALTER TABLE invoices ADD CONSTRAINT invoices_pkey PRIMARY KEY (id); +CREATE TABLE notes (id UUID NOT NULL); +ALTER TABLE ONLY notes ADD PRIMARY KEY(id); +CREATE TABLE reactions (userid TEXT NOT NULL, postid TEXT NOT NULL); +DO $$ +BEGIN + ALTER TABLE reactions DROP CONSTRAINT IF EXISTS reactions_pkey, ADD PRIMARY KEY (postid, userid); +END $$; +CREATE TABLE tags (id UUID NOT NULL); +ALTER TABLE tags ADD CONSTRAINT tags_pkey PRIMARY KEY USING INDEX tags_id_idx;` + if err := Apply(src, "0001.sql", text); err != nil { + t.Fatal(err) + } + if k := src.Table("public", "invoices").PrimaryKey; formatSlice(k) != "[id]" { + t.Errorf("invoices key = %v", k) + } + if k := src.Table("public", "notes").PrimaryKey; formatSlice(k) != "[id]" { + t.Errorf("notes key = %v", k) + } + // The DO body is not run, so its key is not known, and says so. + if r := src.Table("public", "reactions"); !r.KeyUnknown || len(r.PrimaryKey) != 0 { + t.Errorf("reactions: KeyUnknown %v key %v, want unknown and none read", r.KeyUnknown, r.PrimaryKey) + } + if !src.Table("public", "tags").KeyUnknown { + t.Error("tags: PRIMARY KEY USING INDEX names no columns; the key should be unknown") + } + if src.Table("public", "invoices").KeyUnknown { + t.Error("invoices: a key read in full is known") + } +}