diff --git a/README.md b/README.md index adf5d73..594afc9 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,7 @@ points at, with the reason: ```sql ALTER TABLE users - ADD COLUMN ssn_hash TEXT, -- cdclint:ignore schema-before-connector: PII, never streamed + ADD COLUMN ssn_hash TEXT, -- cdclint:ignore schema-before-connector,source-column-not-captured: PII, never streamed ADD COLUMN nickname TEXT; ``` @@ -213,10 +213,12 @@ below. It works in migrations and in sink DDL, with `--` or MySQL's `#`, and names one or more rules separated by commas. The reason after the colon is required, because it is the record of the decision. A marker with no reason, with a rule name that does not exist, or that covers no finding is itself a -warning (`ignore-marker`). A `schema-before-connector` marker is expected to -go quiet once its pull request merges, since that rule judges the change; -to keep the column out of the inventory afterwards too, name both rules: -`-- cdclint:ignore schema-before-connector,source-column-not-captured: PII`. +warning (`ignore-marker`). Name both rules for a column left out of the +stream, as above and as the diff rule's own fix suggests: +`schema-before-connector` only exists in the pull request that adds the +column, and after the merge the same column is `source-column-not-captured` +on every run. A marker naming only the first goes quiet at the merge and the +column comes back into the inventory. Files in, findings out, non-zero exit. No database, no daemon, no credentials. Under a second on a laptop. `--fail-on warning` or `info` raises the bar; diff --git a/corpus/README.md b/corpus/README.md index 98ecd1d..39c5d2f 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -27,6 +27,8 @@ The rest exercise one path each: | `kafka-topic-typo` | a Kafka-engine table naming a topic nothing produces | | `include-list-typo` | an include-list pattern that matches no column, and the read it silently breaks | | `ignore-marker-pii` | the diff rule with a decision recorded: of two columns added and left off the include list, the one whose line says `cdclint:ignore schema-before-connector: PII` is acknowledged and the forgotten one is raised; a trailing marker does not reach the next line | +| `ignore-marker-both-rules` | the marker the diff rule's fix suggests, `schema-before-connector,source-column-not-captured`, in the pull request that adds the column: acknowledged, and the second rule is not reported as covering nothing | +| `ignore-marker-both-rules-merged` | the same migration after the merge (no base): the column stays out of the inventory of uncaptured columns, acknowledged as `source-column-not-captured` | | `ignore-marker-mistakes` | markers used wrong: one with no reason, one naming a rule that does not exist, one covering nothing, each an `ignore-marker` warning; and one on the line above a finding, which acknowledges it | | `golang-migrate-down-files` | Mattermost v10.11.0's `000092_add_createat_to_teammembers.down.sql` drops a column from the wrong table; read in name order it ran just before its up file and deleted `reactions.createat`. Down files are skipped | | `goose-down-section` | a goose file's down section follows its up section; applied whole, every table was created and dropped again. Down sections are skipped | diff --git a/corpus/diff-adds-column-connector-untouched/expected.txt b/corpus/diff-adds-column-connector-untouched/expected.txt index 32e5bf5..5123699 100644 --- a/corpus/diff-adds-column-connector-untouched/expected.txt +++ b/corpus/diff-adds-column-connector-untouched/expected.txt @@ -4,5 +4,5 @@ info source-column-not-captured diff-adds-column-connector-untouched/migrations/ warning schema-before-connector diff-adds-column-connector-untouched/migrations/0035_validation_response_distance.sql:5 this change adds public.report_validations.response_distance_m to a captured table without adding it to column.include.list in diff-adds-column-connector-untouched/connector.json (compared with base) the column will not be in the stream; if a sink is later given it, every row will be the default until a snapshot - fix: add public.report_validations.response_distance_m to column.include.list in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector: + fix: add public.report_validations.response_distance_m to column.include.list in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector,source-column-not-captured: 0 error(s), 1 warning(s), 1 info diff --git a/corpus/diff-connector-captures-one-of-two/expected.txt b/corpus/diff-connector-captures-one-of-two/expected.txt index 3c7fe20..42c1d9b 100644 --- a/corpus/diff-connector-captures-one-of-two/expected.txt +++ b/corpus/diff-connector-captures-one-of-two/expected.txt @@ -1,5 +1,5 @@ warning schema-before-connector diff-connector-captures-one-of-two/migrations/0141_movement_doubt_cleared.sql:7 this change adds public.reports.movement_cleared_by to a captured table without adding it to column.include.list in diff-connector-captures-one-of-two/connector.json (compared with base) the column will not be in the stream; if a sink is later given it, every row will be the default until a snapshot - fix: add public.reports.movement_cleared_by to column.include.list in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector: + fix: add public.reports.movement_cleared_by to column.include.list in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector,source-column-not-captured: 0 error(s), 1 warning(s), 0 info diff --git a/corpus/diff-connector-touched-other-table/expected.txt b/corpus/diff-connector-touched-other-table/expected.txt index b06df4f..ee4a4a3 100644 --- a/corpus/diff-connector-touched-other-table/expected.txt +++ b/corpus/diff-connector-touched-other-table/expected.txt @@ -1,13 +1,13 @@ warning schema-before-connector diff-connector-touched-other-table/migrations/0141_movement_doubt_cleared.sql:7 this change adds public.reports.movement_cleared_at to a captured table without adding it to column.include.list in diff-connector-touched-other-table/connector.json (compared with base) the column will not be in the stream; if a sink is later given it, every row will be the default until a snapshot - fix: add public.reports.movement_cleared_at to column.include.list in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector: + fix: add public.reports.movement_cleared_at to column.include.list in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector,source-column-not-captured: warning schema-before-connector diff-connector-touched-other-table/migrations/0141_movement_doubt_cleared.sql:8 this change adds public.reports.movement_cleared_by to a captured table without adding it to column.include.list in diff-connector-touched-other-table/connector.json (compared with base) the column will not be in the stream; if a sink is later given it, every row will be the default until a snapshot - fix: add public.reports.movement_cleared_by to column.include.list in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector: + fix: add public.reports.movement_cleared_by to column.include.list in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector,source-column-not-captured: warning schema-before-connector diff-connector-touched-other-table/migrations/0141_movement_doubt_cleared.sql:9 this change adds public.reports.movement_clear_reason to a captured table without adding it to column.include.list in diff-connector-touched-other-table/connector.json (compared with base) the column will not be in the stream; if a sink is later given it, every row will be the default until a snapshot - fix: add public.reports.movement_clear_reason to column.include.list in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector: + fix: add public.reports.movement_clear_reason to column.include.list in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector,source-column-not-captured: 0 error(s), 3 warning(s), 0 info diff --git a/corpus/diff-include-list-narrowed/expected.txt b/corpus/diff-include-list-narrowed/expected.txt index 31d7801..4d055e2 100644 --- a/corpus/diff-include-list-narrowed/expected.txt +++ b/corpus/diff-include-list-narrowed/expected.txt @@ -4,5 +4,5 @@ info source-column-not-captured diff-include-list-narrowed/migrations/0033_repor warning schema-before-connector diff-include-list-narrowed/migrations/0035_validation_response_distance.sql:5 this change adds public.report_validations.response_distance_m to a captured table without adding it to column.include.list in diff-include-list-narrowed/connector.json (compared with base) the column will not be in the stream; if a sink is later given it, every row will be the default until a snapshot - fix: add public.report_validations.response_distance_m to column.include.list in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector: + fix: add public.report_validations.response_distance_m to column.include.list in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector,source-column-not-captured: 0 error(s), 1 warning(s), 1 info diff --git a/corpus/ignore-marker-both-rules-merged/connector.json b/corpus/ignore-marker-both-rules-merged/connector.json new file mode 100644 index 0000000..bd44d0c --- /dev/null +++ b/corpus/ignore-marker-both-rules-merged/connector.json @@ -0,0 +1,12 @@ +{ + "name": "app-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "plugin.name": "pgoutput", + "topic.prefix": "app", + "table.include.list": "public.users", + "column.include.list": "public\\.users\\.(id|email)", + "transforms": "unwrap", + "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState" + } +} diff --git a/corpus/ignore-marker-both-rules-merged/expected.txt b/corpus/ignore-marker-both-rules-merged/expected.txt new file mode 100644 index 0000000..ccfaeb2 --- /dev/null +++ b/corpus/ignore-marker-both-rules-merged/expected.txt @@ -0,0 +1,2 @@ +ok: nothing to report beyond what cdclint:ignore acknowledges +acknowledged (cdclint:ignore): source-column-not-captured ignore-marker-both-rules-merged/migrations/0002_users_ssn_hash.sql:4: PII, never streamed diff --git a/corpus/ignore-marker-both-rules-merged/migrations/0001_users.sql b/corpus/ignore-marker-both-rules-merged/migrations/0001_users.sql new file mode 100644 index 0000000..088f383 --- /dev/null +++ b/corpus/ignore-marker-both-rules-merged/migrations/0001_users.sql @@ -0,0 +1,4 @@ +CREATE TABLE users ( + id UUID PRIMARY KEY, + email TEXT NOT NULL +); diff --git a/corpus/ignore-marker-both-rules-merged/migrations/0002_users_ssn_hash.sql b/corpus/ignore-marker-both-rules-merged/migrations/0002_users_ssn_hash.sql new file mode 100644 index 0000000..6b9bef4 --- /dev/null +++ b/corpus/ignore-marker-both-rules-merged/migrations/0002_users_ssn_hash.sql @@ -0,0 +1,4 @@ +-- The same migration after its pull request merged: with no base there is no +-- change to judge, and the column would be in the inventory of uncaptured +-- columns on every run if the marker named only schema-before-connector. +ALTER TABLE users ADD COLUMN ssn_hash TEXT; -- cdclint:ignore schema-before-connector,source-column-not-captured: PII, never streamed diff --git a/corpus/ignore-marker-both-rules-merged/sink/0001_users.sql b/corpus/ignore-marker-both-rules-merged/sink/0001_users.sql new file mode 100644 index 0000000..8ba3d3a --- /dev/null +++ b/corpus/ignore-marker-both-rules-merged/sink/0001_users.sql @@ -0,0 +1,11 @@ +CREATE TABLE kafka_users +( + `id` String, + `email` String +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'app.public.users', + kafka_group_name = 'clickhouse-users', + kafka_format = 'JSONEachRow'; diff --git a/corpus/ignore-marker-both-rules/base/connector.json b/corpus/ignore-marker-both-rules/base/connector.json new file mode 100644 index 0000000..bd44d0c --- /dev/null +++ b/corpus/ignore-marker-both-rules/base/connector.json @@ -0,0 +1,12 @@ +{ + "name": "app-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "plugin.name": "pgoutput", + "topic.prefix": "app", + "table.include.list": "public.users", + "column.include.list": "public\\.users\\.(id|email)", + "transforms": "unwrap", + "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState" + } +} diff --git a/corpus/ignore-marker-both-rules/base/migrations/0001_users.sql b/corpus/ignore-marker-both-rules/base/migrations/0001_users.sql new file mode 100644 index 0000000..088f383 --- /dev/null +++ b/corpus/ignore-marker-both-rules/base/migrations/0001_users.sql @@ -0,0 +1,4 @@ +CREATE TABLE users ( + id UUID PRIMARY KEY, + email TEXT NOT NULL +); diff --git a/corpus/ignore-marker-both-rules/connector.json b/corpus/ignore-marker-both-rules/connector.json new file mode 100644 index 0000000..bd44d0c --- /dev/null +++ b/corpus/ignore-marker-both-rules/connector.json @@ -0,0 +1,12 @@ +{ + "name": "app-source", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "plugin.name": "pgoutput", + "topic.prefix": "app", + "table.include.list": "public.users", + "column.include.list": "public\\.users\\.(id|email)", + "transforms": "unwrap", + "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState" + } +} diff --git a/corpus/ignore-marker-both-rules/expected.txt b/corpus/ignore-marker-both-rules/expected.txt new file mode 100644 index 0000000..e2a989f --- /dev/null +++ b/corpus/ignore-marker-both-rules/expected.txt @@ -0,0 +1,2 @@ +ok: nothing to report beyond what cdclint:ignore acknowledges +acknowledged (cdclint:ignore): schema-before-connector ignore-marker-both-rules/migrations/0002_users_ssn_hash.sql:3: PII, never streamed diff --git a/corpus/ignore-marker-both-rules/migrations/0001_users.sql b/corpus/ignore-marker-both-rules/migrations/0001_users.sql new file mode 100644 index 0000000..088f383 --- /dev/null +++ b/corpus/ignore-marker-both-rules/migrations/0001_users.sql @@ -0,0 +1,4 @@ +CREATE TABLE users ( + id UUID PRIMARY KEY, + email TEXT NOT NULL +); diff --git a/corpus/ignore-marker-both-rules/migrations/0002_users_ssn_hash.sql b/corpus/ignore-marker-both-rules/migrations/0002_users_ssn_hash.sql new file mode 100644 index 0000000..5d1afcc --- /dev/null +++ b/corpus/ignore-marker-both-rules/migrations/0002_users_ssn_hash.sql @@ -0,0 +1,3 @@ +-- The change under review adds ssn_hash and leaves it off the include list on +-- purpose, with the marker the diff rule's own fix suggests: both rules named. +ALTER TABLE users ADD COLUMN ssn_hash TEXT; -- cdclint:ignore schema-before-connector,source-column-not-captured: PII, never streamed diff --git a/corpus/ignore-marker-both-rules/sink/0001_users.sql b/corpus/ignore-marker-both-rules/sink/0001_users.sql new file mode 100644 index 0000000..8ba3d3a --- /dev/null +++ b/corpus/ignore-marker-both-rules/sink/0001_users.sql @@ -0,0 +1,11 @@ +CREATE TABLE kafka_users +( + `id` String, + `email` String +) +ENGINE = Kafka +SETTINGS + kafka_broker_list = '${KAFKA_BROKERS}', + kafka_topic_list = 'app.public.users', + kafka_group_name = 'clickhouse-users', + kafka_format = 'JSONEachRow'; diff --git a/corpus/ignore-marker-pii/expected.txt b/corpus/ignore-marker-pii/expected.txt index 87e1cb6..e06bff2 100644 --- a/corpus/ignore-marker-pii/expected.txt +++ b/corpus/ignore-marker-pii/expected.txt @@ -1,6 +1,6 @@ warning schema-before-connector ignore-marker-pii/migrations/0002_users_profile.sql:6 this change adds public.users.nickname to a captured table without adding it to column.include.list in ignore-marker-pii/connector.json (compared with base) the column will not be in the stream; if a sink is later given it, every row will be the default until a snapshot - fix: add public.users.nickname to column.include.list in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector: + fix: add public.users.nickname to column.include.list in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector,source-column-not-captured: 0 error(s), 1 warning(s), 0 info acknowledged (cdclint:ignore): schema-before-connector ignore-marker-pii/migrations/0002_users_profile.sql:5: PII, never streamed diff --git a/corpus/mysql-diff-adds-column/expected.txt b/corpus/mysql-diff-adds-column/expected.txt index 3ed92d3..4592315 100644 --- a/corpus/mysql-diff-adds-column/expected.txt +++ b/corpus/mysql-diff-adds-column/expected.txt @@ -1,5 +1,5 @@ warning schema-before-connector mysql-diff-adds-column/migrations/V2__orders_coupon.sql:3 this change adds shop.orders.coupon_code to a captured table without adding it to column.include.list in mysql-diff-adds-column/connector.json (compared with base) the column will not be in the stream; if a sink is later given it, every row will be the default until a snapshot - fix: add shop.orders.coupon_code to column.include.list in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector: + fix: add shop.orders.coupon_code to column.include.list in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector,source-column-not-captured: 0 error(s), 1 warning(s), 0 info diff --git a/internal/engine/diff.go b/internal/engine/diff.go index 876daf2..fe6d56f 100644 --- a/internal/engine/diff.go +++ b/internal/engine/diff.go @@ -108,10 +108,16 @@ func schemaBeforeConnector(in *Input, reads []Read) ([]model.Finding, map[string } q := fmt.Sprintf("%s.%s", t.Qualified(), c.Name) raised[strings.ToLower(q)] = true + // The suggested marker names both rules. This rule only exists + // in the pull request that adds the column; once it merges the + // same column is source-column-not-captured on every run, so a + // marker naming only this rule put a deliberately excluded + // column back into the inventory (RefuseRadar #1142, round 1, + // where the pinned script's own example had made that mistake). fs = append(fs, model.Finding{ Rule: "schema-before-connector", Severity: model.Warning, Pos: c.Pos, Message: fmt.Sprintf("this change adds %s to a captured table without adding it to %s in %s (compared with %s)\nthe column will not be in the stream; if a sink is later given it, every row will be the default until a snapshot", q, in.Contract.ColumnListSetting(), in.Contract.Pos().File, short(in.Base.Ref)), - Fix: fmt.Sprintf("%s in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector: ", edit(in.Contract.ColumnListSetting(), q)), + Fix: fmt.Sprintf("%s in the same change, or, if it is left off on purpose, say so on the line that adds it: -- cdclint:ignore schema-before-connector,source-column-not-captured: ", edit(in.Contract.ColumnListSetting(), q)), }) } } diff --git a/internal/ignore/ignore.go b/internal/ignore/ignore.go index 88d2f40..1837c2a 100644 --- a/internal/ignore/ignore.go +++ b/internal/ignore/ignore.go @@ -4,7 +4,7 @@ // where the pull request that made it shows it. // // ALTER TABLE users -// ADD COLUMN ssn_hash TEXT; -- cdclint:ignore schema-before-connector: PII, never streamed +// ADD COLUMN ssn_hash TEXT; -- cdclint:ignore schema-before-connector,source-column-not-captured: PII, never streamed // // A marker names one or more rules and gives a reason after a colon. After // code on the same line it covers that line only; on a line of its own it