From 436708f3c07f9c09e541ab79829f7ff593f242ab Mon Sep 17 00:00:00 2001 From: avison9 Date: Sat, 14 Nov 2026 17:15:43 +0100 Subject: [PATCH 1/2] A Postgres table constraint written UNIQUE(a, b) is no longer read as a column Postgres needs no space between a constraint keyword and its parenthesis, and Mattermost's migrations use that form throughout: UNIQUE(name, deleteat), PRIMARY KEY(id, channelid). The word splitter keeps a parenthesised group glued to the word before it, so the reader saw a column named "UNIQUE(name, deleteat)" (an info line for a column that does not exist) and a word "KEY(id, channelid)" that hid the primary key. The same held for CHECK( on a column and for ALTER TABLE ... ADD UNIQUE( / ADD CHECK(, which added a column. Keywords in the Postgres reader are now compared up to a parenthesis. The helper stays in this reader: the shared ddl.HasPrefixFold is also used by the ClickHouse reader, where a word like Kafka(...) is compared whole. Measured on all 280 of Mattermost v10.11.0's Postgres migration files: nine made-up columns before (channels, emoji, PropertyGroups, roles, schemes, sharedchannelremotes, teams, usergroups twice), none after, and no real column lost. RefuseRadar unchanged at 162 info. Not done here: Postgres folds unquoted identifiers to lower case, and the reader keeps CREATE TABLE PropertyGroups as written. Its own change. --- corpus/README.md | 1 + .../postgres-glued-constraints/connector.json | 10 +++++ .../postgres-glued-constraints/expected.txt | 1 + .../migrations/000006_create_emojis.up.sql | 12 ++++++ .../000037_create_sharedchannelremotes.up.sql | 17 ++++++++ .../sink/0001_mattermost.sql | 28 +++++++++++++ internal/source/postgres/postgres.go | 36 +++++++++++++++-- internal/source/postgres/postgres_test.go | 39 +++++++++++++++++++ 8 files changed, 140 insertions(+), 4 deletions(-) create mode 100644 corpus/postgres-glued-constraints/connector.json create mode 100644 corpus/postgres-glued-constraints/expected.txt create mode 100644 corpus/postgres-glued-constraints/migrations/000006_create_emojis.up.sql create mode 100644 corpus/postgres-glued-constraints/migrations/000037_create_sharedchannelremotes.up.sql create mode 100644 corpus/postgres-glued-constraints/sink/0001_mattermost.sql diff --git a/corpus/README.md b/corpus/README.md index 39c5d2f..a260d75 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -32,6 +32,7 @@ The rest exercise one path each: | `ignore-marker-mistakes` | markers used wrong: one with no reason, one naming a rule that does not exist, one covering nothing, each an `ignore-marker` warning; and one on the line above a finding, which acknowledges it | | `golang-migrate-down-files` | Mattermost v10.11.0's `000092_add_createat_to_teammembers.down.sql` drops a column from the wrong table; read in name order it ran just before its up file and deleted `reactions.createat`. Down files are skipped | | `goose-down-section` | a goose file's down section follows its up section; applied whole, every table was created and dropped again. Down sections are skipped | +| `postgres-glued-constraints` | Mattermost v10.11.0 (000006 emoji, 000037 sharedchannelremotes): `UNIQUE(name, deleteat)` and `PRIMARY KEY(id, channelid)` with no space before the parenthesis are table constraints, not columns named after them; the primary key is kept | | `include-table-no-schema` | Stack Overflow 74103659: `table.include.list` names `ipaddrs` without its schema, the connector runs and no topic appears | | `include-table-glob` | Stack Overflow 51345636: `public.bg_*` written as a shell glob; as a regular expression it names no table | | `include-table-typo` | a misspelled `table.include.list` entry, which Debezium only logs as a warning (debezium/dbz#872) | diff --git a/corpus/postgres-glued-constraints/connector.json b/corpus/postgres-glued-constraints/connector.json new file mode 100644 index 0000000..8603d31 --- /dev/null +++ b/corpus/postgres-glued-constraints/connector.json @@ -0,0 +1,10 @@ +{ + "name": "mattermost-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "plugin.name": "pgoutput", + "topic.prefix": "mm", + "table.include.list": "public.emoji,public.sharedchannelremotes", + "column.include.list": "public.emoji\\.(id|createat|updateat|deleteat|creatorid|name),public.sharedchannelremotes\\.(id|channelid|creatorid|createat|updateat|isinviteaccepted|isinviteconfirmed|remoteid|lastpostupdateat|lastpostid)" + } +} diff --git a/corpus/postgres-glued-constraints/expected.txt b/corpus/postgres-glued-constraints/expected.txt new file mode 100644 index 0000000..cd1f70b --- /dev/null +++ b/corpus/postgres-glued-constraints/expected.txt @@ -0,0 +1 @@ +ok: source, connector and sink agree diff --git a/corpus/postgres-glued-constraints/migrations/000006_create_emojis.up.sql b/corpus/postgres-glued-constraints/migrations/000006_create_emojis.up.sql new file mode 100644 index 0000000..7e07422 --- /dev/null +++ b/corpus/postgres-glued-constraints/migrations/000006_create_emojis.up.sql @@ -0,0 +1,12 @@ +-- Mattermost v10.11.0, server/channels/db/migrations/postgres/ +-- 000006_create_emojis.up.sql, reduced to the table. The table constraint +-- is written UNIQUE(name, deleteat), with no space before the parenthesis. +CREATE TABLE IF NOT EXISTS emoji ( + id VARCHAR(26) PRIMARY KEY, + createat bigint, + updateat bigint, + deleteat bigint, + creatorid VARCHAR(26), + name VARCHAR(64), + UNIQUE(name, deleteat) +); diff --git a/corpus/postgres-glued-constraints/migrations/000037_create_sharedchannelremotes.up.sql b/corpus/postgres-glued-constraints/migrations/000037_create_sharedchannelremotes.up.sql new file mode 100644 index 0000000..dfe2e67 --- /dev/null +++ b/corpus/postgres-glued-constraints/migrations/000037_create_sharedchannelremotes.up.sql @@ -0,0 +1,17 @@ +-- Mattermost v10.11.0, 000037_create_sharedchannelremotes.up.sql, as +-- written: both table constraints are glued to their parenthesis. +CREATE TABLE IF NOT EXISTS sharedchannelremotes ( + id character varying(26) NOT NULL, + channelid character varying(26) NOT NULL, + creatorid character varying(26), + createat bigint, + updateat bigint, + isinviteaccepted boolean, + isinviteconfirmed boolean, + remoteid character varying(26), + PRIMARY KEY(id, channelid), + UNIQUE(channelid, remoteid) +); + +ALTER TABLE sharedchannelremotes ADD COLUMN IF NOT EXISTS LastPostUpdateAt bigint; +ALTER TABLE sharedchannelremotes ADD COLUMN IF NOT EXISTS LastPostId character varying(26); diff --git a/corpus/postgres-glued-constraints/sink/0001_mattermost.sql b/corpus/postgres-glued-constraints/sink/0001_mattermost.sql new file mode 100644 index 0000000..ccac0ec --- /dev/null +++ b/corpus/postgres-glued-constraints/sink/0001_mattermost.sql @@ -0,0 +1,28 @@ +-- Invented sink for the entry: two Kafka-engine tables reading a few of the +-- captured columns. The entry is about the source reader, not the sink. +CREATE TABLE kafka_emoji +( + `id` String, + `name` String, + `deleteat` Int64 +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'mm.public.emoji', + kafka_group_name = 'clickhouse-emoji', + kafka_format = 'JSONEachRow'; + +CREATE TABLE kafka_sharedchannelremotes +( + `id` String, + `channelid` String, + `remoteid` String, + `lastpostupdateat` Int64 +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'mm.public.sharedchannelremotes', + kafka_group_name = 'clickhouse-sharedchannelremotes', + kafka_format = 'JSONEachRow'; diff --git a/internal/source/postgres/postgres.go b/internal/source/postgres/postgres.go index a21e83e..a4733b1 100644 --- a/internal/source/postgres/postgres.go +++ b/internal/source/postgres/postgres.go @@ -117,9 +117,9 @@ func createTable(src *model.Source, text string, pos model.Pos) { if len(iw) == 0 { continue } - switch strings.ToUpper(iw[0]) { + switch keyword(iw[0]) { case "CONSTRAINT", "PRIMARY", "UNIQUE", "CHECK", "FOREIGN", "EXCLUDE", "LIKE": - if ddl.HasPrefixFold(iw, "PRIMARY", "KEY") { + if hasKeywords(iw, "PRIMARY", "KEY") { t.PrimaryKey = keyColumns(item) } if ddl.HasPrefixFold(iw, "CONSTRAINT") && containsFold(iw, "PRIMARY") { @@ -145,7 +145,7 @@ func createTable(src *model.Source, text string, pos model.Pos) { // the first constraint keyword. func typeWords(w []string) []string { for i, x := range w { - switch strings.ToUpper(x) { + switch keyword(x) { case "NOT", "NULL", "DEFAULT", "PRIMARY", "REFERENCES", "UNIQUE", "CHECK", "GENERATED", "CONSTRAINT", "COLLATE": return w[:i] } @@ -165,6 +165,34 @@ func keyColumns(item string) []string { return cols } +// keyword is a word as a keyword: upper case, and cut at an opening +// parenthesis. ddl.Words keeps a parenthesised group glued to the word +// before it, and Postgres needs no space there: Mattermost writes +// UNIQUE(channelid, remoteid) and PRIMARY KEY(id, channelid), which were +// read as a column named "UNIQUE(channelid, remoteid)" and as no primary +// key. Kept to this reader: in the ClickHouse reader a word such as +// Kafka(...) is compared whole on purpose. +func keyword(w string) string { + if i := strings.IndexByte(w, '('); i >= 0 { + w = w[:i] + } + return strings.ToUpper(w) +} + +// hasKeywords is ddl.HasPrefixFold for keywords that may be glued to a +// parenthesis. +func hasKeywords(w []string, kws ...string) bool { + if len(w) < len(kws) { + return false + } + for i, kw := range kws { + if keyword(w[i]) != kw { + return false + } + } + return true +} + func containsFold(w []string, kw string) bool { for _, x := range w { if strings.EqualFold(x, kw) { @@ -199,7 +227,7 @@ func alterTable(src *model.Source, text string, w []string, pos model.Pos) { continue } switch { - case ddl.HasPrefixFold(aw, "ADD", "COLUMN"), ddl.HasPrefixFold(aw, "ADD") && !ddl.HasPrefixFold(aw, "ADD", "CONSTRAINT") && !ddl.HasPrefixFold(aw, "ADD", "PRIMARY") && !ddl.HasPrefixFold(aw, "ADD", "UNIQUE") && !ddl.HasPrefixFold(aw, "ADD", "CHECK") && !ddl.HasPrefixFold(aw, "ADD", "FOREIGN") && !ddl.HasPrefixFold(aw, "ADD", "EXCLUDE"): + case ddl.HasPrefixFold(aw, "ADD", "COLUMN"), ddl.HasPrefixFold(aw, "ADD") && !hasKeywords(aw, "ADD", "CONSTRAINT") && !hasKeywords(aw, "ADD", "PRIMARY") && !hasKeywords(aw, "ADD", "UNIQUE") && !hasKeywords(aw, "ADD", "CHECK") && !hasKeywords(aw, "ADD", "FOREIGN") && !hasKeywords(aw, "ADD", "EXCLUDE"): j := 1 if strings.EqualFold(aw[j], "COLUMN") { j++ diff --git a/internal/source/postgres/postgres_test.go b/internal/source/postgres/postgres_test.go index dd2994b..5b26099 100644 --- a/internal/source/postgres/postgres_test.go +++ b/internal/source/postgres/postgres_test.go @@ -103,3 +103,42 @@ func TestColumnsAnAlterAddsAreOnTheirOwnLines(t *testing.T) { } } } + +func TestAConstraintGluedToItsParenthesisIsNotAColumn(t *testing.T) { + // Mattermost 000037_create_sharedchannelremotes.up.sql writes + // PRIMARY KEY(id, channelid) and UNIQUE(channelid, remoteid) with no + // space: the word splitter keeps "UNIQUE(channelid, remoteid)" as one + // word, which was read as a column of that name, and "KEY(id, + // channelid)", which hid the primary key. + src := &model.Source{} + text := `CREATE TABLE IF NOT EXISTS sharedchannelremotes ( + id character varying(26) NOT NULL, + channelid character varying(26) NOT NULL, + remoteid character varying(26), + price int CHECK(price > 0), + PRIMARY KEY(id, channelid), + UNIQUE(channelid, remoteid), + CHECK(remoteid <> '') +); +ALTER TABLE sharedchannelremotes ADD UNIQUE(remoteid); +ALTER TABLE sharedchannelremotes ADD CHECK(price < 100); +ALTER TABLE sharedchannelremotes ADD COLUMN note TEXT;` + if err := Apply(src, "000037.up.sql", text); err != nil { + t.Fatal(err) + } + r := src.Table("public", "sharedchannelremotes") + if r == nil { + t.Fatal("table missing") + } + var names []string + for _, c := range r.Columns { + names = append(names, c.Name+":"+c.Type) + } + want := "[id:character varying(26) channelid:character varying(26) remoteid:character varying(26) price:int note:TEXT]" + if got := formatSlice(names); got != want { + t.Errorf("columns = %s, want %s", got, want) + } + if formatSlice(r.PrimaryKey) != "[id channelid]" { + t.Errorf("primary key = %v, want [id channelid]", r.PrimaryKey) + } +} From d3807b653a52da2a0b65272e38abdd7d82df6726 Mon Sep 17 00:00:00 2001 From: avison9 Date: Sat, 14 Nov 2026 17:23:38 +0100 Subject: [PATCH 2/2] replica-identity warns when a captured Postgres table has no replica identity A table on REPLICA IDENTITY NOTHING, or on DEFAULT with no primary key, has no replica identity. Both consequences are documented. Postgres (logical-replication-publication): such a table "cannot support UPDATE or DELETE operations when included in a publication replicating these actions", which is every pgoutput connector, so the application's own writes fail. Debezium (PostgreSQL connector, Replica identity): with DEFAULT and no primary key "the connector does not emit UPDATE or DELETE events for that table", and with NOTHING its events carry no previous values. The finding says which one applies, from plugin.name (decoderbufs when unset, the documented default). The connector's replica.identity.autoset.values overrides the database on the tables it captures, so it is read and wins; when it decides, the finding points at the connector file. Two pairs that disagree about one table, or a pair that cannot be read, leave the identity unknown and the table unjudged: the documentation does not say which pair wins. A warning, not an error: a table that only takes inserts loses nothing, and the fix says how to record that with cdclint:ignore replica-identity. To keep it from judging keys it cannot see, the Postgres reader now reads ALTER TABLE ... ADD [CONSTRAINT x] PRIMARY KEY (a key added later), and marks a table's key unknown when a DO block alters it or the key is PRIMARY KEY USING INDEX. Mattermost adds reactions' key in a DO block; running DO bodies is its own change. USING INDEX identities are not judged: whether the index still exists is not in the files. Two older corpus entries had been reduced without the keys their sources have; they now carry them, and their expectations do not change. golang-migrate-down-files gets Mattermost's real teammembers key and reactions' DO block. flatten-bare-columns gets REPLICA IDENTITY FULL: the discussion's sample message has a whole before row on a keyless table, which only FULL produces. Rejected: judging what the sink reads from the before image (Flatten on the envelope reading before.). It is the README's longer promise for this rule, needs the transform model to say which before fields a sink reads, and is its own change. --- README.md | 2 +- corpus/README.md | 6 ++ .../migrations/0001_testradiomeas.sql | 5 + .../migrations/0001_create_tables.up.sql | 18 +++- .../connector.json | 9 ++ .../expected.txt | 1 + .../migrations/0001_audit_events.sql | 7 ++ .../sink/0001_audit_events.sql | 14 +++ .../connector.json | 9 ++ .../expected.txt | 5 + .../migrations/0001_orders.sql | 6 ++ .../sink/0001_orders.sql | 13 +++ .../connector.json | 8 ++ .../expected.txt | 1 + .../migrations/0001_invoices.sql | 7 ++ .../sink/0001_invoices.sql | 13 +++ .../connector.json | 8 ++ .../expected.txt | 1 + .../migrations/000016_create_reactions.up.sql | 35 +++++++ .../sink/0001_reactions.sql | 14 +++ .../connector.json | 8 ++ .../expected.txt | 5 + .../migrations/0001_audit_events.sql | 7 ++ .../sink/0001_audit_events.sql | 14 +++ .../connector.json | 9 ++ .../expected.txt | 5 + .../migrations/0001_sessions.sql | 8 ++ .../sink/0001_sessions.sql | 14 +++ internal/capture/debezium/debezium.go | 79 +++++++++++++++ internal/capture/debezium/debezium_test.go | 42 +++++++- internal/engine/engine.go | 1 + internal/engine/replica.go | 96 +++++++++++++++++++ internal/engine/rules.go | 1 + internal/model/model.go | 7 +- internal/source/postgres/postgres.go | 28 ++++++ internal/source/postgres/postgres_test.go | 34 +++++++ 36 files changed, 533 insertions(+), 7 deletions(-) create mode 100644 corpus/replica-identity-autoset-full/connector.json create mode 100644 corpus/replica-identity-autoset-full/expected.txt create mode 100644 corpus/replica-identity-autoset-full/migrations/0001_audit_events.sql create mode 100644 corpus/replica-identity-autoset-full/sink/0001_audit_events.sql create mode 100644 corpus/replica-identity-autoset-nothing/connector.json create mode 100644 corpus/replica-identity-autoset-nothing/expected.txt create mode 100644 corpus/replica-identity-autoset-nothing/migrations/0001_orders.sql create mode 100644 corpus/replica-identity-autoset-nothing/sink/0001_orders.sql create mode 100644 corpus/replica-identity-key-added-later/connector.json create mode 100644 corpus/replica-identity-key-added-later/expected.txt create mode 100644 corpus/replica-identity-key-added-later/migrations/0001_invoices.sql create mode 100644 corpus/replica-identity-key-added-later/sink/0001_invoices.sql create mode 100644 corpus/replica-identity-key-in-do-block/connector.json create mode 100644 corpus/replica-identity-key-in-do-block/expected.txt create mode 100644 corpus/replica-identity-key-in-do-block/migrations/000016_create_reactions.up.sql create mode 100644 corpus/replica-identity-key-in-do-block/sink/0001_reactions.sql create mode 100644 corpus/replica-identity-no-primary-key/connector.json create mode 100644 corpus/replica-identity-no-primary-key/expected.txt create mode 100644 corpus/replica-identity-no-primary-key/migrations/0001_audit_events.sql create mode 100644 corpus/replica-identity-no-primary-key/sink/0001_audit_events.sql create mode 100644 corpus/replica-identity-nothing-pgoutput/connector.json create mode 100644 corpus/replica-identity-nothing-pgoutput/expected.txt create mode 100644 corpus/replica-identity-nothing-pgoutput/migrations/0001_sessions.sql create mode 100644 corpus/replica-identity-nothing-pgoutput/sink/0001_sessions.sql create mode 100644 internal/engine/replica.go diff --git a/README.md b/README.md index 7f91126..7d5d37a 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ anything running. | `sink-column-flattened` | a sink table names its columns as the source does while a `Flatten` transform on the way delivers the envelope as `after.`, so every column stays at its default | v0.3 | | `mv-column-match` | ClickHouse streaming materialized views match by name, refreshable ones by position. ClickHouse 25.4 and later reject a streaming view that writes a column the target lacks when it is created; a refreshable view's order mismatch was loud on 24.8 and is silent on 26.8 | v0.1 | | `schema-before-connector` | this change adds a column to a captured table and leaves it out of the stream without deciding to, the trap itself, judged on the diff (warning: leaving PII off is right, so it asks for the decision) | v0.2 | -| `replica-identity` | a captured table's replica identity cannot supply what the sink reads | next | +| `replica-identity` | a captured Postgres table has no replica identity (`NOTHING`, or `DEFAULT` with no primary key, after the connector's `replica.identity.autoset.values`): through `pgoutput` Postgres rejects the application's `UPDATE` and `DELETE` on it, through `decoderbufs` Debezium streams only its inserts (warning: an insert-only table loses nothing, and a `cdclint:ignore` says so) | next | | `migration-numbering` | duplicate or gapped migration prefixes | next | Findings name the file, the column, the rule and the fix: diff --git a/corpus/README.md b/corpus/README.md index a260d75..b2a2384 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -36,6 +36,12 @@ The rest exercise one path each: | `include-table-no-schema` | Stack Overflow 74103659: `table.include.list` names `ipaddrs` without its schema, the connector runs and no topic appears | | `include-table-glob` | Stack Overflow 51345636: `public.bg_*` written as a shell glob; as a regular expression it names no table | | `include-table-typo` | a misspelled `table.include.list` entry, which Debezium only logs as a warning (debezium/dbz#872) | +| `replica-identity-no-primary-key` | a captured table with no primary key on `REPLICA IDENTITY DEFAULT`, read through the default `decoderbufs`: Debezium emits its inserts only | +| `replica-identity-nothing-pgoutput` | a keyed table set to `REPLICA IDENTITY NOTHING`, read through `pgoutput`: Postgres rejects `UPDATE` and `DELETE` on a published table with no replica identity | +| `replica-identity-autoset-nothing` | the connector's `replica.identity.autoset.values` sets `NOTHING` on a keyed table; the finding points at the connector, which is what to edit | +| `replica-identity-autoset-full` | the same keyless table as `replica-identity-no-primary-key`, with the connector setting `FULL`: nothing to report | +| `replica-identity-key-added-later` | the primary key arrives in a later `ALTER TABLE ... ADD CONSTRAINT ... PRIMARY KEY`: nothing to report | +| `replica-identity-key-in-do-block` | Mattermost v10.11.0 `000016_create_reactions.up.sql`, as written: the key is added inside a `DO` block, which cdclint does not run, so it makes no finding rather than guess | | `flatten-bare-columns` | ClickHouse/clickhouse-kafka-connect discussion 182: a `Flatten` transform on the Debezium connector sends `after.`, the ClickHouse table names its columns as Postgres does, and every change lands as a row of zeros | | `flatten-envelope-columns` | ClickHouse's own Postgres CDC example (ClickHouse/examples `cdc/postgresql` at ae417ae): `before.` and `after.` columns resolve to the source columns, so the capture rules see `postcode2`, which the include list added for this entry leaves out. The sink connector config there has no `connector.class`; it is added so the file names its sink | | `flatten-after-unwrap` | a guard: unwrapping first leaves a flat row, so a `Flatten` after it renames nothing | diff --git a/corpus/flatten-bare-columns/migrations/0001_testradiomeas.sql b/corpus/flatten-bare-columns/migrations/0001_testradiomeas.sql index 1d44320..596870f 100644 --- a/corpus/flatten-bare-columns/migrations/0001_testradiomeas.sql +++ b/corpus/flatten-bare-columns/migrations/0001_testradiomeas.sql @@ -9,3 +9,8 @@ CREATE TABLE testradiomeas ( "measNetNoCoverage" INTEGER, "measNetType" TEXT ); + +-- The discussion's sample message carries a whole "before" row (measDate, +-- measLongitude, ...) on a table with no primary key, which only REPLICA +-- IDENTITY FULL produces: with DEFAULT the connector would emit inserts only. +ALTER TABLE testradiomeas REPLICA IDENTITY FULL; diff --git a/corpus/golang-migrate-down-files/migrations/0001_create_tables.up.sql b/corpus/golang-migrate-down-files/migrations/0001_create_tables.up.sql index 99d27dc..44e118d 100644 --- a/corpus/golang-migrate-down-files/migrations/0001_create_tables.up.sql +++ b/corpus/golang-migrate-down-files/migrations/0001_create_tables.up.sql @@ -1,10 +1,20 @@ +-- Reduced from Mattermost v10.11.0 000002 and 000016, keys as Mattermost +-- declares them: teammembers' in its CREATE TABLE, reactions' in a DO block +-- (cdclint does not run DO bodies, so it does not judge reactions' key). CREATE TABLE reactions ( - userid VARCHAR(26) NOT NULL, - postid VARCHAR(26) NOT NULL, - createat BIGINT + userid VARCHAR(26) NOT NULL, + postid VARCHAR(26) NOT NULL, + emojiname VARCHAR(64) NOT NULL, + createat BIGINT ); +DO $$ +BEGIN + ALTER TABLE reactions ADD PRIMARY KEY (postid, userid, emojiname); +END $$; + CREATE TABLE teammembers ( teamid VARCHAR(26) NOT NULL, - userid VARCHAR(26) NOT NULL + userid VARCHAR(26) NOT NULL, + PRIMARY KEY (teamid, userid) ); diff --git a/corpus/replica-identity-autoset-full/connector.json b/corpus/replica-identity-autoset-full/connector.json new file mode 100644 index 0000000..23d100d --- /dev/null +++ b/corpus/replica-identity-autoset-full/connector.json @@ -0,0 +1,9 @@ +{ + "name": "app-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "topic.prefix": "app", + "table.include.list": "public.audit_events", + "replica.identity.autoset.values": "public.audit_.*:FULL" + } +} diff --git a/corpus/replica-identity-autoset-full/expected.txt b/corpus/replica-identity-autoset-full/expected.txt new file mode 100644 index 0000000..cd1f70b --- /dev/null +++ b/corpus/replica-identity-autoset-full/expected.txt @@ -0,0 +1 @@ +ok: source, connector and sink agree diff --git a/corpus/replica-identity-autoset-full/migrations/0001_audit_events.sql b/corpus/replica-identity-autoset-full/migrations/0001_audit_events.sql new file mode 100644 index 0000000..64298cc --- /dev/null +++ b/corpus/replica-identity-autoset-full/migrations/0001_audit_events.sql @@ -0,0 +1,7 @@ +-- Invented for the entry: a table created without a primary key, as tables +-- meant only to be appended to often are, then captured like the rest. +CREATE TABLE audit_events ( + actor_id UUID NOT NULL, + action TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/corpus/replica-identity-autoset-full/sink/0001_audit_events.sql b/corpus/replica-identity-autoset-full/sink/0001_audit_events.sql new file mode 100644 index 0000000..13b77f4 --- /dev/null +++ b/corpus/replica-identity-autoset-full/sink/0001_audit_events.sql @@ -0,0 +1,14 @@ +-- Invented sink for the entry: a Kafka-engine table reading the captured +-- table. The entry is about the source table's replica identity. +CREATE TABLE kafka_audit_events +( + `actor_id` String, + `action` String, + `created_at` String +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'app.public.audit_events', + kafka_group_name = 'clickhouse-audit_events', + kafka_format = 'JSONEachRow'; diff --git a/corpus/replica-identity-autoset-nothing/connector.json b/corpus/replica-identity-autoset-nothing/connector.json new file mode 100644 index 0000000..d26094b --- /dev/null +++ b/corpus/replica-identity-autoset-nothing/connector.json @@ -0,0 +1,9 @@ +{ + "name": "app-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "topic.prefix": "app", + "table.include.list": "public.orders", + "replica.identity.autoset.values": "public.orders:NOTHING" + } +} diff --git a/corpus/replica-identity-autoset-nothing/expected.txt b/corpus/replica-identity-autoset-nothing/expected.txt new file mode 100644 index 0000000..9564c18 --- /dev/null +++ b/corpus/replica-identity-autoset-nothing/expected.txt @@ -0,0 +1,5 @@ +warning replica-identity replica-identity-autoset-nothing/connector.json + public.orders is captured with REPLICA IDENTITY NOTHING, set by replica.identity.autoset.values in replica-identity-autoset-nothing/connector.json, so it has no replica identity + Debezium's update and delete events for it carry no previous values, so a delete has no before to say which row went + fix: take public.orders out of replica.identity.autoset.values, or set it to DEFAULT (it has a primary key) or FULL; if it only ever takes inserts, say so on the line that creates it: -- cdclint:ignore replica-identity: append-only +0 error(s), 1 warning(s), 0 info diff --git a/corpus/replica-identity-autoset-nothing/migrations/0001_orders.sql b/corpus/replica-identity-autoset-nothing/migrations/0001_orders.sql new file mode 100644 index 0000000..9c16211 --- /dev/null +++ b/corpus/replica-identity-autoset-nothing/migrations/0001_orders.sql @@ -0,0 +1,6 @@ +-- Invented for the entry: a keyed table whose replica identity the +-- connector overrides with replica.identity.autoset.values. +CREATE TABLE orders ( + id UUID PRIMARY KEY, + status TEXT NOT NULL +); diff --git a/corpus/replica-identity-autoset-nothing/sink/0001_orders.sql b/corpus/replica-identity-autoset-nothing/sink/0001_orders.sql new file mode 100644 index 0000000..c6b7395 --- /dev/null +++ b/corpus/replica-identity-autoset-nothing/sink/0001_orders.sql @@ -0,0 +1,13 @@ +-- Invented sink for the entry: a Kafka-engine table reading the captured +-- table. The entry is about the source table's replica identity. +CREATE TABLE kafka_orders +( + `id` String, + `status` String +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'app.public.orders', + kafka_group_name = 'clickhouse-orders', + kafka_format = 'JSONEachRow'; diff --git a/corpus/replica-identity-key-added-later/connector.json b/corpus/replica-identity-key-added-later/connector.json new file mode 100644 index 0000000..0f2574d --- /dev/null +++ b/corpus/replica-identity-key-added-later/connector.json @@ -0,0 +1,8 @@ +{ + "name": "app-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "topic.prefix": "app", + "table.include.list": "public.invoices" + } +} diff --git a/corpus/replica-identity-key-added-later/expected.txt b/corpus/replica-identity-key-added-later/expected.txt new file mode 100644 index 0000000..cd1f70b --- /dev/null +++ b/corpus/replica-identity-key-added-later/expected.txt @@ -0,0 +1 @@ +ok: source, connector and sink agree diff --git a/corpus/replica-identity-key-added-later/migrations/0001_invoices.sql b/corpus/replica-identity-key-added-later/migrations/0001_invoices.sql new file mode 100644 index 0000000..4073063 --- /dev/null +++ b/corpus/replica-identity-key-added-later/migrations/0001_invoices.sql @@ -0,0 +1,7 @@ +-- Invented for the entry: the primary key is added after the table, by +-- ALTER TABLE, as migrations that split constraints out often do. +CREATE TABLE invoices ( + id UUID NOT NULL, + total NUMERIC NOT NULL +); +ALTER TABLE invoices ADD CONSTRAINT invoices_pkey PRIMARY KEY (id); diff --git a/corpus/replica-identity-key-added-later/sink/0001_invoices.sql b/corpus/replica-identity-key-added-later/sink/0001_invoices.sql new file mode 100644 index 0000000..023d83a --- /dev/null +++ b/corpus/replica-identity-key-added-later/sink/0001_invoices.sql @@ -0,0 +1,13 @@ +-- Invented sink for the entry: a Kafka-engine table reading the captured +-- table. The entry is about the source table's replica identity. +CREATE TABLE kafka_invoices +( + `id` String, + `total` String +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'app.public.invoices', + kafka_group_name = 'clickhouse-invoices', + kafka_format = 'JSONEachRow'; diff --git a/corpus/replica-identity-key-in-do-block/connector.json b/corpus/replica-identity-key-in-do-block/connector.json new file mode 100644 index 0000000..7e75db5 --- /dev/null +++ b/corpus/replica-identity-key-in-do-block/connector.json @@ -0,0 +1,8 @@ +{ + "name": "mm-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "topic.prefix": "mm", + "table.include.list": "public.reactions" + } +} diff --git a/corpus/replica-identity-key-in-do-block/expected.txt b/corpus/replica-identity-key-in-do-block/expected.txt new file mode 100644 index 0000000..cd1f70b --- /dev/null +++ b/corpus/replica-identity-key-in-do-block/expected.txt @@ -0,0 +1 @@ +ok: source, connector and sink agree diff --git a/corpus/replica-identity-key-in-do-block/migrations/000016_create_reactions.up.sql b/corpus/replica-identity-key-in-do-block/migrations/000016_create_reactions.up.sql new file mode 100644 index 0000000..b4d320f --- /dev/null +++ b/corpus/replica-identity-key-in-do-block/migrations/000016_create_reactions.up.sql @@ -0,0 +1,35 @@ +-- Mattermost v10.11.0, server/channels/db/migrations/postgres/ +-- 000016_create_reactions.up.sql, as written: the table is created with no +-- primary key and the DO block below adds one. cdclint does not run DO +-- bodies, so it cannot know the key, and makes no replica-identity finding. +CREATE TABLE IF NOT EXISTS reactions( + userid VARCHAR(26) NOT NULL, + postid VARCHAR(26) NOT NULL, + emojiname VARCHAR(64) NOT NULL, + createat bigint +); + +ALTER TABLE reactions ADD COLUMN IF NOT EXISTS updateat bigint; +ALTER TABLE reactions ADD COLUMN IF NOT EXISTS deleteat bigint; + +DO $$ +<> +DECLARE + existing_index text; +BEGIN + SELECT string_agg(a.attname, ',') INTO existing_index + FROM pg_constraint AS c + CROSS JOIN + (SELECT unnest(conkey) FROM pg_constraint WHERE conrelid = 'reactions'::regclass AND contype='p') AS cols(colnum) + INNER JOIN pg_attribute AS a ON a.attrelid = c.conrelid AND cols.colnum = a.attnum + WHERE c.contype = 'p' + AND c.conrelid = 'reactions'::regclass; + + IF COALESCE (existing_index, '') <> text('postid,userid,emojiname') THEN + ALTER TABLE reactions + DROP CONSTRAINT IF EXISTS reactions_pkey, + ADD PRIMARY KEY (postid, userid, emojiname); + END IF; +END alter_pk $$; + +ALTER TABLE reactions ADD COLUMN IF NOT EXISTS remoteid VARCHAR(26); diff --git a/corpus/replica-identity-key-in-do-block/sink/0001_reactions.sql b/corpus/replica-identity-key-in-do-block/sink/0001_reactions.sql new file mode 100644 index 0000000..1b8f1f8 --- /dev/null +++ b/corpus/replica-identity-key-in-do-block/sink/0001_reactions.sql @@ -0,0 +1,14 @@ +-- Invented sink for the entry: a Kafka-engine table reading the captured +-- table. The entry is about the source table's replica identity. +CREATE TABLE kafka_reactions +( + `userid` String, + `postid` String, + `emojiname` String +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'mm.public.reactions', + kafka_group_name = 'clickhouse-reactions', + kafka_format = 'JSONEachRow'; diff --git a/corpus/replica-identity-no-primary-key/connector.json b/corpus/replica-identity-no-primary-key/connector.json new file mode 100644 index 0000000..1799bcd --- /dev/null +++ b/corpus/replica-identity-no-primary-key/connector.json @@ -0,0 +1,8 @@ +{ + "name": "app-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "topic.prefix": "app", + "table.include.list": "public.audit_events" + } +} diff --git a/corpus/replica-identity-no-primary-key/expected.txt b/corpus/replica-identity-no-primary-key/expected.txt new file mode 100644 index 0000000..f3c4cc6 --- /dev/null +++ b/corpus/replica-identity-no-primary-key/expected.txt @@ -0,0 +1,5 @@ +warning replica-identity replica-identity-no-primary-key/migrations/0001_audit_events.sql:3 + public.audit_events is captured with REPLICA IDENTITY DEFAULT and no primary key, so it has no replica identity + Debezium emits only its inserts: updates and deletes never reach the stream, and the sink keeps every row as it was first written + fix: ALTER TABLE public.audit_events REPLICA IDENTITY FULL; in a new migration, or give it a primary key; if it only ever takes inserts, say so on the line that creates it: -- cdclint:ignore replica-identity: append-only +0 error(s), 1 warning(s), 0 info diff --git a/corpus/replica-identity-no-primary-key/migrations/0001_audit_events.sql b/corpus/replica-identity-no-primary-key/migrations/0001_audit_events.sql new file mode 100644 index 0000000..64298cc --- /dev/null +++ b/corpus/replica-identity-no-primary-key/migrations/0001_audit_events.sql @@ -0,0 +1,7 @@ +-- Invented for the entry: a table created without a primary key, as tables +-- meant only to be appended to often are, then captured like the rest. +CREATE TABLE audit_events ( + actor_id UUID NOT NULL, + action TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/corpus/replica-identity-no-primary-key/sink/0001_audit_events.sql b/corpus/replica-identity-no-primary-key/sink/0001_audit_events.sql new file mode 100644 index 0000000..13b77f4 --- /dev/null +++ b/corpus/replica-identity-no-primary-key/sink/0001_audit_events.sql @@ -0,0 +1,14 @@ +-- Invented sink for the entry: a Kafka-engine table reading the captured +-- table. The entry is about the source table's replica identity. +CREATE TABLE kafka_audit_events +( + `actor_id` String, + `action` String, + `created_at` String +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'app.public.audit_events', + kafka_group_name = 'clickhouse-audit_events', + kafka_format = 'JSONEachRow'; diff --git a/corpus/replica-identity-nothing-pgoutput/connector.json b/corpus/replica-identity-nothing-pgoutput/connector.json new file mode 100644 index 0000000..22730f5 --- /dev/null +++ b/corpus/replica-identity-nothing-pgoutput/connector.json @@ -0,0 +1,9 @@ +{ + "name": "app-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "topic.prefix": "app", + "table.include.list": "public.sessions", + "plugin.name": "pgoutput" + } +} diff --git a/corpus/replica-identity-nothing-pgoutput/expected.txt b/corpus/replica-identity-nothing-pgoutput/expected.txt new file mode 100644 index 0000000..3a1ad64 --- /dev/null +++ b/corpus/replica-identity-nothing-pgoutput/expected.txt @@ -0,0 +1,5 @@ +warning replica-identity replica-identity-nothing-pgoutput/migrations/0001_sessions.sql:3 + public.sessions is captured with REPLICA IDENTITY NOTHING, so it has no replica identity + Postgres rejects UPDATE and DELETE on a table with no replica identity while a publication replicates them, and this connector reads through one (plugin.name pgoutput): the application's own writes to it fail + fix: ALTER TABLE public.sessions REPLICA IDENTITY DEFAULT; in a new migration (it has a primary key), or FULL; if it only ever takes inserts, say so on the line that creates it: -- cdclint:ignore replica-identity: append-only +0 error(s), 1 warning(s), 0 info diff --git a/corpus/replica-identity-nothing-pgoutput/migrations/0001_sessions.sql b/corpus/replica-identity-nothing-pgoutput/migrations/0001_sessions.sql new file mode 100644 index 0000000..1d470d4 --- /dev/null +++ b/corpus/replica-identity-nothing-pgoutput/migrations/0001_sessions.sql @@ -0,0 +1,8 @@ +-- Invented for the entry: a keyed table later set to REPLICA IDENTITY +-- NOTHING, read through pgoutput, so through a publication. +CREATE TABLE sessions ( + id UUID PRIMARY KEY, + user_id UUID NOT NULL, + expires_at TIMESTAMPTZ NOT NULL +); +ALTER TABLE sessions REPLICA IDENTITY NOTHING; diff --git a/corpus/replica-identity-nothing-pgoutput/sink/0001_sessions.sql b/corpus/replica-identity-nothing-pgoutput/sink/0001_sessions.sql new file mode 100644 index 0000000..b34d2ed --- /dev/null +++ b/corpus/replica-identity-nothing-pgoutput/sink/0001_sessions.sql @@ -0,0 +1,14 @@ +-- Invented sink for the entry: a Kafka-engine table reading the captured +-- table. The entry is about the source table's replica identity. +CREATE TABLE kafka_sessions +( + `id` String, + `user_id` String, + `expires_at` String +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'app.public.sessions', + kafka_group_name = 'clickhouse-sessions', + kafka_format = 'JSONEachRow'; diff --git a/internal/capture/debezium/debezium.go b/internal/capture/debezium/debezium.go index ab9e0b8..7fea13d 100644 --- a/internal/capture/debezium/debezium.go +++ b/internal/capture/debezium/debezium.go @@ -40,12 +40,22 @@ type Contract struct { columnPatterns []string tablePatterns []string routers []router + autoset []autoset + autosetErr bool shape model.Shape Config map[string]string Class string Name string } +// autoset is one pair of replica.identity.autoset.values: a regular +// expression over schema.table and the identity the connector sets on the +// tables it captures that match it. +type autoset struct { + re *regexp.Regexp + identity model.ReplicaIdentity +} + type router struct { re *regexp.Regexp replacement string @@ -128,6 +138,7 @@ func Parse(b []byte) (*Contract, error) { if c.columnListKey == "" { c.columnListKey = "column.include.list" } + c.autoset, c.autosetErr = autosets(cfg["replica.identity.autoset.values"]) if c.tableListKey == "" { c.tableListKey = "table.include.list" } @@ -333,3 +344,71 @@ func (c *Contract) ColumnPatterns() []string { return c.columnPatterns } // reason as columns. func (c *Contract) TablePatterns() []string { return c.tablePatterns } func (c *Contract) TableListSetting() string { return c.tableListKey } + +// autosets reads replica.identity.autoset.values, documented as +// comma-separated ":" pairs, identity one of DEFAULT, +// FULL, NOTHING or INDEX . A pair it cannot read makes the whole +// setting unreadable (bad), so the replica-identity rule makes no finding +// rather than judge a table by a setting it misread. +func autosets(v string) (out []autoset, bad bool) { + for _, pair := range strings.Split(v, ",") { + pair = strings.TrimSpace(pair) + if pair == "" { + continue + } + i := strings.LastIndex(pair, ":") + if i <= 0 { + return nil, true + } + // Matched like the include lists: whole name, ignoring case. + re, err := regexp.Compile("(?i)^(?:" + strings.TrimSpace(pair[:i]) + ")$") + if err != nil { + return nil, true + } + var id model.ReplicaIdentity + switch value := strings.ToUpper(strings.TrimSpace(pair[i+1:])); { + case value == "DEFAULT": + id = model.ReplicaDefault + case value == "FULL": + id = model.ReplicaFull + case value == "NOTHING": + id = model.ReplicaNothing + case strings.HasPrefix(value, "INDEX "): + id = model.ReplicaIndex + default: + return nil, true + } + out = append(out, autoset{re, id}) + } + return out, false +} + +// ReplicaIdentityOverride is the identity replica.identity.autoset.values +// gives a captured table. set is false when no pair names the table. ok is +// false when the setting cannot be read or two pairs give the table +// different identities: the documentation does not say which wins. +func (c *Contract) ReplicaIdentityOverride(schema, table string) (id model.ReplicaIdentity, set, ok bool) { + if c.autosetErr { + return "", true, false + } + q := schema + "." + table + for _, a := range c.autoset { + if !a.re.MatchString(q) { + continue + } + if set && a.identity != id { + return "", true, false + } + id, set = a.identity, true + } + return id, set, true +} + +// Plugin is plugin.name, the logical decoding plug-in; decoderbufs when +// unset, Debezium's documented default. +func (c *Contract) Plugin() string { + if p := strings.TrimSpace(c.Config["plugin.name"]); p != "" { + return strings.ToLower(p) + } + return "decoderbufs" +} diff --git a/internal/capture/debezium/debezium_test.go b/internal/capture/debezium/debezium_test.go index 6d86232..4962c95 100644 --- a/internal/capture/debezium/debezium_test.go +++ b/internal/capture/debezium/debezium_test.go @@ -1,6 +1,10 @@ package debezium -import "testing" +import ( + "testing" + + "github.com/avison9/cdclint/internal/model" +) func TestIncludeListsAndTopics(t *testing.T) { c, err := Parse([]byte(`{"name":"pg","config":{ @@ -55,3 +59,39 @@ func TestExcludeList(t *testing.T) { t.Errorf("column list key = %q", c.ColumnListSetting()) } } + +func TestReplicaIdentityAutosetValues(t *testing.T) { + // The documented form: comma-separated : pairs. + c, err := Parse([]byte(`{"config":{"replica.identity.autoset.values":"public.audit_.*:FULL, public.orders:NOTHING, s2.t3:INDEX idx_name"}}`)) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + schema, table string + id model.ReplicaIdentity + set bool + }{ + {"public", "audit_events", model.ReplicaFull, true}, + {"public", "ORDERS", model.ReplicaNothing, true}, + {"s2", "t3", model.ReplicaIndex, true}, + {"public", "users", "", false}, + } { + id, set, ok := c.ReplicaIdentityOverride(tc.schema, tc.table) + if !ok || set != tc.set || id != tc.id { + t.Errorf("%s.%s: id %q set %v ok %v, want %q %v true", tc.schema, tc.table, id, set, ok, tc.id, tc.set) + } + } + if c.Plugin() != "decoderbufs" { + t.Errorf("plugin = %q, want the documented default decoderbufs", c.Plugin()) + } + // Two pairs that disagree about one table, and a pair that cannot be + // read, both leave the identity unknown rather than guessed. + two, _ := Parse([]byte(`{"config":{"replica.identity.autoset.values":"public.*:FULL,public.orders:NOTHING"}}`)) + if _, set, ok := two.ReplicaIdentityOverride("public", "orders"); !set || ok { + t.Errorf("conflicting pairs: set %v ok %v, want set and not ok", set, ok) + } + bad, _ := Parse([]byte(`{"config":{"replica.identity.autoset.values":"public.orders:EVERYTHING"}}`)) + if _, _, ok := bad.ReplicaIdentityOverride("public", "users"); ok { + t.Error("an unreadable setting should not be trusted for any table") + } +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go index c89acd5..02dc32f 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -199,6 +199,7 @@ func Run(in *Input) []model.Finding { fs = append(fs, sourceColumnNotCaptured(in, reads, raised)...) fs = append(fs, capturedColumnMissing(in)...) fs = append(fs, capturedTableMissing(in)...) + fs = append(fs, replicaIdentity(in)...) fs = append(fs, mvColumnMatch(in)...) model.Sort(fs) return fs diff --git a/internal/engine/replica.go b/internal/engine/replica.go new file mode 100644 index 0000000..96f7920 --- /dev/null +++ b/internal/engine/replica.go @@ -0,0 +1,96 @@ +package engine + +import ( + "fmt" + + "github.com/avison9/cdclint/internal/model" +) + +// ReplicaIdentityContract is implemented by contracts that can override a +// table's replica identity (Debezium's replica.identity.autoset.values) +// and name the logical decoding plug-in they read through. +type ReplicaIdentityContract interface { + ReplicaIdentityOverride(schema, table string) (id model.ReplicaIdentity, set, ok bool) + Plugin() string +} + +// replicaIdentity raises a captured Postgres table with no replica +// identity: REPLICA IDENTITY NOTHING, or DEFAULT with no primary key. +// Both consequences are documented, not inferred. Postgres: "Tables with a +// replica identity defined as NOTHING, DEFAULT without a primary key, or +// USING INDEX with a dropped index, cannot support UPDATE or DELETE +// operations when included in a publication replicating these actions" +// (logical-replication-publication), which is the pgoutput case. Debezium +// (PostgreSQL connector, "Replica identity"): with DEFAULT and no primary +// key "the connector does not emit UPDATE or DELETE events for that +// table", and with NOTHING its update and delete events "do not contain +// any information about the previous value of any table column". +// +// USING INDEX is not judged: whether the index still exists is not in the +// files. Neither is a table whose key may be set where the reader cannot +// see (KeyUnknown), nor one whose connector override cannot be read. A +// warning, not an error: a table that only ever takes inserts loses +// nothing, and the fix says how to record that. +func replicaIdentity(in *Input) []model.Finding { + rc, _ := in.Contract.(ReplicaIdentityContract) + var fs []model.Finding + for _, t := range in.Source.Tables { + // MySQL tables carry no replica identity; the setting is Postgres'. + if t.ReplicaIdentity == "" || t.KeyUnknown || !in.Contract.CapturesTable(t.Schema, t.Name) { + continue + } + id, pos, overridden := t.ReplicaIdentity, t.Pos, false + if rc != nil { + o, set, ok := rc.ReplicaIdentityOverride(t.Schema, t.Name) + if !ok { + continue + } + if set { + // The connector sets it on every start, so the + // override is what the stream sees, and the + // connector file is what to edit. + id, pos, overridden = o, in.Contract.Pos(), true + } + } + keyed := len(t.PrimaryKey) > 0 + var what string + switch { + case id == model.ReplicaNothing: + what = "REPLICA IDENTITY NOTHING" + case id == model.ReplicaDefault && !keyed: + what = "REPLICA IDENTITY DEFAULT and no primary key" + default: + continue + } + q := t.Qualified() + if overridden { + what += fmt.Sprintf(", set by replica.identity.autoset.values in %s", in.Contract.Pos().File) + } + var consequence string + switch { + case rc != nil && rc.Plugin() == "pgoutput": + consequence = "Postgres rejects UPDATE and DELETE on a table with no replica identity while a publication replicates them, and this connector reads through one (plugin.name pgoutput): the application's own writes to it fail" + case id == model.ReplicaNothing: + consequence = "Debezium's update and delete events for it carry no previous values, so a delete has no before to say which row went" + default: + consequence = "Debezium emits only its inserts: updates and deletes never reach the stream, and the sink keeps every row as it was first written" + } + var fix string + switch { + case overridden && keyed: + fix = fmt.Sprintf("take %s out of replica.identity.autoset.values, or set it to DEFAULT (it has a primary key) or FULL", q) + case overridden: + fix = fmt.Sprintf("set %s to FULL in replica.identity.autoset.values, or give it a primary key", q) + case keyed: + fix = fmt.Sprintf("ALTER TABLE %s REPLICA IDENTITY DEFAULT; in a new migration (it has a primary key), or FULL", q) + default: + fix = fmt.Sprintf("ALTER TABLE %s REPLICA IDENTITY FULL; in a new migration, or give it a primary key", q) + } + fs = append(fs, model.Finding{ + Rule: "replica-identity", Severity: model.Warning, Pos: pos, + Message: fmt.Sprintf("%s is captured with %s, so it has no replica identity\n%s", q, what, consequence), + Fix: fix + "; if it only ever takes inserts, say so on the line that creates it: -- cdclint:ignore replica-identity: append-only", + }) + } + return fs +} diff --git a/internal/engine/rules.go b/internal/engine/rules.go index 06ed784..3d39e7b 100644 --- a/internal/engine/rules.go +++ b/internal/engine/rules.go @@ -11,6 +11,7 @@ var Rules = []string{ "source-column-not-captured", "captured-column-missing", "captured-table-missing", + "replica-identity", "topic-table-mapping", "sink-column-flattened", "mv-column-match", diff --git a/internal/model/model.go b/internal/model/model.go index df8cbb3..48fadb7 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -54,7 +54,12 @@ type Table struct { Columns []Column PrimaryKey []string ReplicaIdentity ReplicaIdentity - Pos Pos + // KeyUnknown is set when a statement the reader does not run may have + // given the table its primary key or replica identity: a DO block that + // alters it (Mattermost adds reactions' key that way), or PRIMARY KEY + // USING INDEX. Rules that depend on the key make no finding then. + KeyUnknown bool + Pos Pos } // Qualified is schema.name, the form Debezium matches include lists against. diff --git a/internal/source/postgres/postgres.go b/internal/source/postgres/postgres.go index a4733b1..64d7aa2 100644 --- a/internal/source/postgres/postgres.go +++ b/internal/source/postgres/postgres.go @@ -10,6 +10,7 @@ package postgres import ( "fmt" + "regexp" "strings" "github.com/avison9/cdclint/internal/ddl" @@ -64,11 +65,30 @@ func Apply(src *model.Source, file, text string) error { alterTable(src, st.Text, w, pos) case ddl.HasPrefixFold(w, "DROP", "TABLE"): dropTable(src, w) + case ddl.HasPrefixFold(w, "DO"): + doBlock(src, st.Text) } } return nil } +// alterInDo finds the tables a DO block's body alters. +var alterInDo = regexp.MustCompile(`(?i)\bALTER\s+TABLE\s+(?:ONLY\s+)?(?:IF\s+EXISTS\s+)?((?:"[^"]+"|[\w$]+)(?:\.(?:"[^"]+"|[\w$]+))?)`) + +// doBlock does not run a DO block: its body is procedural, often guarded +// by a condition on the catalog. It marks the tables the body alters as +// having an unknown key, so a rule that depends on the key stays quiet +// rather than judge a table whose key may be added there (Mattermost +// 000016_create_reactions.up.sql adds reactions' primary key in one). +func doBlock(src *model.Source, text string) { + for _, m := range alterInDo.FindAllStringSubmatch(text, -1) { + schema, table := splitQualified(m[1]) + if t := src.Table(schema, table); t != nil { + t.KeyUnknown = true + } + } +} + func splitQualified(name string) (schema, table string) { parts := ddl.SplitName(name) if len(parts) >= 2 { @@ -268,6 +288,14 @@ func alterTable(src *model.Source, text string, w []string, pos model.Pos) { c.Pos = pos } } + case hasKeywords(aw, "ADD", "PRIMARY", "KEY"), hasKeywords(aw, "ADD", "CONSTRAINT") && len(aw) > 3 && hasKeywords(aw[3:], "PRIMARY", "KEY"): + // A key added after the table, as migrations that split + // constraints out do. USING INDEX names no columns here. + if cols := keyColumns(a); len(cols) > 0 { + t.PrimaryKey = cols + } else { + t.KeyUnknown = true + } case ddl.HasPrefixFold(aw, "RENAME", "TO"): if len(aw) >= 3 { t.Name = ddl.Unquote(aw[2]) diff --git a/internal/source/postgres/postgres_test.go b/internal/source/postgres/postgres_test.go index 5b26099..7bbf528 100644 --- a/internal/source/postgres/postgres_test.go +++ b/internal/source/postgres/postgres_test.go @@ -142,3 +142,37 @@ ALTER TABLE sharedchannelremotes ADD COLUMN note TEXT;` t.Errorf("primary key = %v, want [id channelid]", r.PrimaryKey) } } + +func TestAKeyAddedLaterOrInADoBlock(t *testing.T) { + src := &model.Source{} + text := `CREATE TABLE invoices (id UUID NOT NULL, total NUMERIC); +ALTER TABLE invoices ADD CONSTRAINT invoices_pkey PRIMARY KEY (id); +CREATE TABLE notes (id UUID NOT NULL); +ALTER TABLE ONLY notes ADD PRIMARY KEY(id); +CREATE TABLE reactions (userid TEXT NOT NULL, postid TEXT NOT NULL); +DO $$ +BEGIN + ALTER TABLE reactions DROP CONSTRAINT IF EXISTS reactions_pkey, ADD PRIMARY KEY (postid, userid); +END $$; +CREATE TABLE tags (id UUID NOT NULL); +ALTER TABLE tags ADD CONSTRAINT tags_pkey PRIMARY KEY USING INDEX tags_id_idx;` + if err := Apply(src, "0001.sql", text); err != nil { + t.Fatal(err) + } + if k := src.Table("public", "invoices").PrimaryKey; formatSlice(k) != "[id]" { + t.Errorf("invoices key = %v", k) + } + if k := src.Table("public", "notes").PrimaryKey; formatSlice(k) != "[id]" { + t.Errorf("notes key = %v", k) + } + // The DO body is not run, so its key is not known, and says so. + if r := src.Table("public", "reactions"); !r.KeyUnknown || len(r.PrimaryKey) != 0 { + t.Errorf("reactions: KeyUnknown %v key %v, want unknown and none read", r.KeyUnknown, r.PrimaryKey) + } + if !src.Table("public", "tags").KeyUnknown { + t.Error("tags: PRIMARY KEY USING INDEX names no columns; the key should be unknown") + } + if src.Table("public", "invoices").KeyUnknown { + t.Error("invoices: a key read in full is known") + } +}