diff --git a/corpus/README.md b/corpus/README.md index 39c5d2f..a260d75 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -32,6 +32,7 @@ The rest exercise one path each: | `ignore-marker-mistakes` | markers used wrong: one with no reason, one naming a rule that does not exist, one covering nothing, each an `ignore-marker` warning; and one on the line above a finding, which acknowledges it | | `golang-migrate-down-files` | Mattermost v10.11.0's `000092_add_createat_to_teammembers.down.sql` drops a column from the wrong table; read in name order it ran just before its up file and deleted `reactions.createat`. Down files are skipped | | `goose-down-section` | a goose file's down section follows its up section; applied whole, every table was created and dropped again. Down sections are skipped | +| `postgres-glued-constraints` | Mattermost v10.11.0 (000006 emoji, 000037 sharedchannelremotes): `UNIQUE(name, deleteat)` and `PRIMARY KEY(id, channelid)` with no space before the parenthesis are table constraints, not columns named after them; the primary key is kept | | `include-table-no-schema` | Stack Overflow 74103659: `table.include.list` names `ipaddrs` without its schema, the connector runs and no topic appears | | `include-table-glob` | Stack Overflow 51345636: `public.bg_*` written as a shell glob; as a regular expression it names no table | | `include-table-typo` | a misspelled `table.include.list` entry, which Debezium only logs as a warning (debezium/dbz#872) | diff --git a/corpus/postgres-glued-constraints/connector.json b/corpus/postgres-glued-constraints/connector.json new file mode 100644 index 0000000..8603d31 --- /dev/null +++ b/corpus/postgres-glued-constraints/connector.json @@ -0,0 +1,10 @@ +{ + "name": "mattermost-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "plugin.name": "pgoutput", + "topic.prefix": "mm", + "table.include.list": "public.emoji,public.sharedchannelremotes", + "column.include.list": "public.emoji\\.(id|createat|updateat|deleteat|creatorid|name),public.sharedchannelremotes\\.(id|channelid|creatorid|createat|updateat|isinviteaccepted|isinviteconfirmed|remoteid|lastpostupdateat|lastpostid)" + } +} diff --git a/corpus/postgres-glued-constraints/expected.txt b/corpus/postgres-glued-constraints/expected.txt new file mode 100644 index 0000000..cd1f70b --- /dev/null +++ b/corpus/postgres-glued-constraints/expected.txt @@ -0,0 +1 @@ +ok: source, connector and sink agree diff --git a/corpus/postgres-glued-constraints/migrations/000006_create_emojis.up.sql b/corpus/postgres-glued-constraints/migrations/000006_create_emojis.up.sql new file mode 100644 index 0000000..7e07422 --- /dev/null +++ b/corpus/postgres-glued-constraints/migrations/000006_create_emojis.up.sql @@ -0,0 +1,12 @@ +-- Mattermost v10.11.0, server/channels/db/migrations/postgres/ +-- 000006_create_emojis.up.sql, reduced to the table. The table constraint +-- is written UNIQUE(name, deleteat), with no space before the parenthesis. +CREATE TABLE IF NOT EXISTS emoji ( + id VARCHAR(26) PRIMARY KEY, + createat bigint, + updateat bigint, + deleteat bigint, + creatorid VARCHAR(26), + name VARCHAR(64), + UNIQUE(name, deleteat) +); diff --git a/corpus/postgres-glued-constraints/migrations/000037_create_sharedchannelremotes.up.sql b/corpus/postgres-glued-constraints/migrations/000037_create_sharedchannelremotes.up.sql new file mode 100644 index 0000000..dfe2e67 --- /dev/null +++ b/corpus/postgres-glued-constraints/migrations/000037_create_sharedchannelremotes.up.sql @@ -0,0 +1,17 @@ +-- Mattermost v10.11.0, 000037_create_sharedchannelremotes.up.sql, as +-- written: both table constraints are glued to their parenthesis. +CREATE TABLE IF NOT EXISTS sharedchannelremotes ( + id character varying(26) NOT NULL, + channelid character varying(26) NOT NULL, + creatorid character varying(26), + createat bigint, + updateat bigint, + isinviteaccepted boolean, + isinviteconfirmed boolean, + remoteid character varying(26), + PRIMARY KEY(id, channelid), + UNIQUE(channelid, remoteid) +); + +ALTER TABLE sharedchannelremotes ADD COLUMN IF NOT EXISTS LastPostUpdateAt bigint; +ALTER TABLE sharedchannelremotes ADD COLUMN IF NOT EXISTS LastPostId character varying(26); diff --git a/corpus/postgres-glued-constraints/sink/0001_mattermost.sql b/corpus/postgres-glued-constraints/sink/0001_mattermost.sql new file mode 100644 index 0000000..ccac0ec --- /dev/null +++ b/corpus/postgres-glued-constraints/sink/0001_mattermost.sql @@ -0,0 +1,28 @@ +-- Invented sink for the entry: two Kafka-engine tables reading a few of the +-- captured columns. The entry is about the source reader, not the sink. +CREATE TABLE kafka_emoji +( + `id` String, + `name` String, + `deleteat` Int64 +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'mm.public.emoji', + kafka_group_name = 'clickhouse-emoji', + kafka_format = 'JSONEachRow'; + +CREATE TABLE kafka_sharedchannelremotes +( + `id` String, + `channelid` String, + `remoteid` String, + `lastpostupdateat` Int64 +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'mm.public.sharedchannelremotes', + kafka_group_name = 'clickhouse-sharedchannelremotes', + kafka_format = 'JSONEachRow'; diff --git a/internal/source/postgres/postgres.go b/internal/source/postgres/postgres.go index a21e83e..a4733b1 100644 --- a/internal/source/postgres/postgres.go +++ b/internal/source/postgres/postgres.go @@ -117,9 +117,9 @@ func createTable(src *model.Source, text string, pos model.Pos) { if len(iw) == 0 { continue } - switch strings.ToUpper(iw[0]) { + switch keyword(iw[0]) { case "CONSTRAINT", "PRIMARY", "UNIQUE", "CHECK", "FOREIGN", "EXCLUDE", "LIKE": - if ddl.HasPrefixFold(iw, "PRIMARY", "KEY") { + if hasKeywords(iw, "PRIMARY", "KEY") { t.PrimaryKey = keyColumns(item) } if ddl.HasPrefixFold(iw, "CONSTRAINT") && containsFold(iw, "PRIMARY") { @@ -145,7 +145,7 @@ func createTable(src *model.Source, text string, pos model.Pos) { // the first constraint keyword. func typeWords(w []string) []string { for i, x := range w { - switch strings.ToUpper(x) { + switch keyword(x) { case "NOT", "NULL", "DEFAULT", "PRIMARY", "REFERENCES", "UNIQUE", "CHECK", "GENERATED", "CONSTRAINT", "COLLATE": return w[:i] } @@ -165,6 +165,34 @@ func keyColumns(item string) []string { return cols } +// keyword is a word as a keyword: upper case, and cut at an opening +// parenthesis. ddl.Words keeps a parenthesised group glued to the word +// before it, and Postgres needs no space there: Mattermost writes +// UNIQUE(channelid, remoteid) and PRIMARY KEY(id, channelid), which were +// read as a column named "UNIQUE(channelid, remoteid)" and as no primary +// key. Kept to this reader: in the ClickHouse reader a word such as +// Kafka(...) is compared whole on purpose. +func keyword(w string) string { + if i := strings.IndexByte(w, '('); i >= 0 { + w = w[:i] + } + return strings.ToUpper(w) +} + +// hasKeywords is ddl.HasPrefixFold for keywords that may be glued to a +// parenthesis. +func hasKeywords(w []string, kws ...string) bool { + if len(w) < len(kws) { + return false + } + for i, kw := range kws { + if keyword(w[i]) != kw { + return false + } + } + return true +} + func containsFold(w []string, kw string) bool { for _, x := range w { if strings.EqualFold(x, kw) { @@ -199,7 +227,7 @@ func alterTable(src *model.Source, text string, w []string, pos model.Pos) { continue } switch { - case ddl.HasPrefixFold(aw, "ADD", "COLUMN"), ddl.HasPrefixFold(aw, "ADD") && !ddl.HasPrefixFold(aw, "ADD", "CONSTRAINT") && !ddl.HasPrefixFold(aw, "ADD", "PRIMARY") && !ddl.HasPrefixFold(aw, "ADD", "UNIQUE") && !ddl.HasPrefixFold(aw, "ADD", "CHECK") && !ddl.HasPrefixFold(aw, "ADD", "FOREIGN") && !ddl.HasPrefixFold(aw, "ADD", "EXCLUDE"): + case ddl.HasPrefixFold(aw, "ADD", "COLUMN"), ddl.HasPrefixFold(aw, "ADD") && !hasKeywords(aw, "ADD", "CONSTRAINT") && !hasKeywords(aw, "ADD", "PRIMARY") && !hasKeywords(aw, "ADD", "UNIQUE") && !hasKeywords(aw, "ADD", "CHECK") && !hasKeywords(aw, "ADD", "FOREIGN") && !hasKeywords(aw, "ADD", "EXCLUDE"): j := 1 if strings.EqualFold(aw[j], "COLUMN") { j++ diff --git a/internal/source/postgres/postgres_test.go b/internal/source/postgres/postgres_test.go index dd2994b..5b26099 100644 --- a/internal/source/postgres/postgres_test.go +++ b/internal/source/postgres/postgres_test.go @@ -103,3 +103,42 @@ func TestColumnsAnAlterAddsAreOnTheirOwnLines(t *testing.T) { } } } + +func TestAConstraintGluedToItsParenthesisIsNotAColumn(t *testing.T) { + // Mattermost 000037_create_sharedchannelremotes.up.sql writes + // PRIMARY KEY(id, channelid) and UNIQUE(channelid, remoteid) with no + // space: the word splitter keeps "UNIQUE(channelid, remoteid)" as one + // word, which was read as a column of that name, and "KEY(id, + // channelid)", which hid the primary key. + src := &model.Source{} + text := `CREATE TABLE IF NOT EXISTS sharedchannelremotes ( + id character varying(26) NOT NULL, + channelid character varying(26) NOT NULL, + remoteid character varying(26), + price int CHECK(price > 0), + PRIMARY KEY(id, channelid), + UNIQUE(channelid, remoteid), + CHECK(remoteid <> '') +); +ALTER TABLE sharedchannelremotes ADD UNIQUE(remoteid); +ALTER TABLE sharedchannelremotes ADD CHECK(price < 100); +ALTER TABLE sharedchannelremotes ADD COLUMN note TEXT;` + if err := Apply(src, "000037.up.sql", text); err != nil { + t.Fatal(err) + } + r := src.Table("public", "sharedchannelremotes") + if r == nil { + t.Fatal("table missing") + } + var names []string + for _, c := range r.Columns { + names = append(names, c.Name+":"+c.Type) + } + want := "[id:character varying(26) channelid:character varying(26) remoteid:character varying(26) price:int note:TEXT]" + if got := formatSlice(names); got != want { + t.Errorf("columns = %s, want %s", got, want) + } + if formatSlice(r.PrimaryKey) != "[id channelid]" { + t.Errorf("primary key = %v, want [id channelid]", r.PrimaryKey) + } +}