Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions corpus/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ The rest exercise one path each:
| `ignore-marker-mistakes` | markers used wrong: one with no reason, one naming a rule that does not exist, one covering nothing, each an `ignore-marker` warning; and one on the line above a finding, which acknowledges it |
| `golang-migrate-down-files` | Mattermost v10.11.0's `000092_add_createat_to_teammembers.down.sql` drops a column from the wrong table; read in name order it ran just before its up file and deleted `reactions.createat`. Down files are skipped |
| `goose-down-section` | a goose file's down section follows its up section; applied whole, every table was created and dropped again. Down sections are skipped |
| `postgres-glued-constraints` | Mattermost v10.11.0 (000006 emoji, 000037 sharedchannelremotes): `UNIQUE(name, deleteat)` and `PRIMARY KEY(id, channelid)` with no space before the parenthesis are table constraints, not columns named after them; the primary key is kept |
| `include-table-no-schema` | Stack Overflow 74103659: `table.include.list` names `ipaddrs` without its schema, the connector runs and no topic appears |
| `include-table-glob` | Stack Overflow 51345636: `public.bg_*` written as a shell glob; as a regular expression it names no table |
| `include-table-typo` | a misspelled `table.include.list` entry, which Debezium only logs as a warning (debezium/dbz#872) |
Expand Down
10 changes: 10 additions & 0 deletions corpus/postgres-glued-constraints/connector.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"name": "mattermost-source",
"config": {
"connector.class": "io.debezium.connector.postgresql.PostgresConnector",
"plugin.name": "pgoutput",
"topic.prefix": "mm",
"table.include.list": "public.emoji,public.sharedchannelremotes",
"column.include.list": "public.emoji\\.(id|createat|updateat|deleteat|creatorid|name),public.sharedchannelremotes\\.(id|channelid|creatorid|createat|updateat|isinviteaccepted|isinviteconfirmed|remoteid|lastpostupdateat|lastpostid)"
}
}
1 change: 1 addition & 0 deletions corpus/postgres-glued-constraints/expected.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ok: source, connector and sink agree
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
-- Mattermost v10.11.0, server/channels/db/migrations/postgres/
-- 000006_create_emojis.up.sql, reduced to the table. The table constraint
-- is written UNIQUE(name, deleteat), with no space before the parenthesis.
CREATE TABLE IF NOT EXISTS emoji (
id VARCHAR(26) PRIMARY KEY,
createat bigint,
updateat bigint,
deleteat bigint,
creatorid VARCHAR(26),
name VARCHAR(64),
UNIQUE(name, deleteat)
);
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
-- Mattermost v10.11.0, 000037_create_sharedchannelremotes.up.sql, as
-- written: both table constraints are glued to their parenthesis.
CREATE TABLE IF NOT EXISTS sharedchannelremotes (
id character varying(26) NOT NULL,
channelid character varying(26) NOT NULL,
creatorid character varying(26),
createat bigint,
updateat bigint,
isinviteaccepted boolean,
isinviteconfirmed boolean,
remoteid character varying(26),
PRIMARY KEY(id, channelid),
UNIQUE(channelid, remoteid)
);

ALTER TABLE sharedchannelremotes ADD COLUMN IF NOT EXISTS LastPostUpdateAt bigint;
ALTER TABLE sharedchannelremotes ADD COLUMN IF NOT EXISTS LastPostId character varying(26);
28 changes: 28 additions & 0 deletions corpus/postgres-glued-constraints/sink/0001_mattermost.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
-- Invented sink for the entry: two Kafka-engine tables reading a few of the
-- captured columns. The entry is about the source reader, not the sink.
CREATE TABLE kafka_emoji
(
`id` String,
`name` String,
`deleteat` Int64
)
ENGINE = Kafka
SETTINGS
kafka_broker_list = '${KAFKA_BROKERS}',
kafka_topic_list = 'mm.public.emoji',
kafka_group_name = 'clickhouse-emoji',
kafka_format = 'JSONEachRow';

CREATE TABLE kafka_sharedchannelremotes
(
`id` String,
`channelid` String,
`remoteid` String,
`lastpostupdateat` Int64
)
ENGINE = Kafka
SETTINGS
kafka_broker_list = '${KAFKA_BROKERS}',
kafka_topic_list = 'mm.public.sharedchannelremotes',
kafka_group_name = 'clickhouse-sharedchannelremotes',
kafka_format = 'JSONEachRow';
36 changes: 32 additions & 4 deletions internal/source/postgres/postgres.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,9 @@ func createTable(src *model.Source, text string, pos model.Pos) {
if len(iw) == 0 {
continue
}
switch strings.ToUpper(iw[0]) {
switch keyword(iw[0]) {
case "CONSTRAINT", "PRIMARY", "UNIQUE", "CHECK", "FOREIGN", "EXCLUDE", "LIKE":
if ddl.HasPrefixFold(iw, "PRIMARY", "KEY") {
if hasKeywords(iw, "PRIMARY", "KEY") {
t.PrimaryKey = keyColumns(item)
}
if ddl.HasPrefixFold(iw, "CONSTRAINT") && containsFold(iw, "PRIMARY") {
Expand All @@ -145,7 +145,7 @@ func createTable(src *model.Source, text string, pos model.Pos) {
// the first constraint keyword.
func typeWords(w []string) []string {
for i, x := range w {
switch strings.ToUpper(x) {
switch keyword(x) {
case "NOT", "NULL", "DEFAULT", "PRIMARY", "REFERENCES", "UNIQUE", "CHECK", "GENERATED", "CONSTRAINT", "COLLATE":
return w[:i]
}
Expand All @@ -165,6 +165,34 @@ func keyColumns(item string) []string {
return cols
}

// keyword is a word as a keyword: upper case, and cut at an opening
// parenthesis. ddl.Words keeps a parenthesised group glued to the word
// before it, and Postgres needs no space there: Mattermost writes
// UNIQUE(channelid, remoteid) and PRIMARY KEY(id, channelid), which were
// read as a column named "UNIQUE(channelid, remoteid)" and as no primary
// key. Kept to this reader: in the ClickHouse reader a word such as
// Kafka(...) is compared whole on purpose.
func keyword(w string) string {
if i := strings.IndexByte(w, '('); i >= 0 {
w = w[:i]
}
return strings.ToUpper(w)
}

// hasKeywords is ddl.HasPrefixFold for keywords that may be glued to a
// parenthesis.
func hasKeywords(w []string, kws ...string) bool {
if len(w) < len(kws) {
return false
}
for i, kw := range kws {
if keyword(w[i]) != kw {
return false
}
}
return true
}

func containsFold(w []string, kw string) bool {
for _, x := range w {
if strings.EqualFold(x, kw) {
Expand Down Expand Up @@ -199,7 +227,7 @@ func alterTable(src *model.Source, text string, w []string, pos model.Pos) {
continue
}
switch {
case ddl.HasPrefixFold(aw, "ADD", "COLUMN"), ddl.HasPrefixFold(aw, "ADD") && !ddl.HasPrefixFold(aw, "ADD", "CONSTRAINT") && !ddl.HasPrefixFold(aw, "ADD", "PRIMARY") && !ddl.HasPrefixFold(aw, "ADD", "UNIQUE") && !ddl.HasPrefixFold(aw, "ADD", "CHECK") && !ddl.HasPrefixFold(aw, "ADD", "FOREIGN") && !ddl.HasPrefixFold(aw, "ADD", "EXCLUDE"):
case ddl.HasPrefixFold(aw, "ADD", "COLUMN"), ddl.HasPrefixFold(aw, "ADD") && !hasKeywords(aw, "ADD", "CONSTRAINT") && !hasKeywords(aw, "ADD", "PRIMARY") && !hasKeywords(aw, "ADD", "UNIQUE") && !hasKeywords(aw, "ADD", "CHECK") && !hasKeywords(aw, "ADD", "FOREIGN") && !hasKeywords(aw, "ADD", "EXCLUDE"):
j := 1
if strings.EqualFold(aw[j], "COLUMN") {
j++
Expand Down
39 changes: 39 additions & 0 deletions internal/source/postgres/postgres_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,3 +103,42 @@ func TestColumnsAnAlterAddsAreOnTheirOwnLines(t *testing.T) {
}
}
}

func TestAConstraintGluedToItsParenthesisIsNotAColumn(t *testing.T) {
// Mattermost 000037_create_sharedchannelremotes.up.sql writes
// PRIMARY KEY(id, channelid) and UNIQUE(channelid, remoteid) with no
// space: the word splitter keeps "UNIQUE(channelid, remoteid)" as one
// word, which was read as a column of that name, and "KEY(id,
// channelid)", which hid the primary key.
src := &model.Source{}
text := `CREATE TABLE IF NOT EXISTS sharedchannelremotes (
id character varying(26) NOT NULL,
channelid character varying(26) NOT NULL,
remoteid character varying(26),
price int CHECK(price > 0),
PRIMARY KEY(id, channelid),
UNIQUE(channelid, remoteid),
CHECK(remoteid <> '')
);
ALTER TABLE sharedchannelremotes ADD UNIQUE(remoteid);
ALTER TABLE sharedchannelremotes ADD CHECK(price < 100);
ALTER TABLE sharedchannelremotes ADD COLUMN note TEXT;`
if err := Apply(src, "000037.up.sql", text); err != nil {
t.Fatal(err)
}
r := src.Table("public", "sharedchannelremotes")
if r == nil {
t.Fatal("table missing")
}
var names []string
for _, c := range r.Columns {
names = append(names, c.Name+":"+c.Type)
}
want := "[id:character varying(26) channelid:character varying(26) remoteid:character varying(26) price:int note:TEXT]"
if got := formatSlice(names); got != want {
t.Errorf("columns = %s, want %s", got, want)
}
if formatSlice(r.PrimaryKey) != "[id channelid]" {
t.Errorf("primary key = %v, want [id channelid]", r.PrimaryKey)
}
}