From f903238464baf836923e8c5141c82de5cd712abc Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Mon, 6 Jul 2026 21:51:30 +0300 Subject: [PATCH 01/48] postgres_cdc: add multi-schema support --- internal/impl/postgresql/input_pg_stream.go | 56 +++++- internal/impl/postgresql/integration_test.go | 166 +++++++++++++++++- .../impl/postgresql/pglogicalstream/config.go | 6 +- .../pglogicalstream/logical_stream.go | 19 +- .../postgresql/pglogicalstream/pglogrepl.go | 72 +++++--- .../pglogicalstream/schema_resolver.go | 129 ++++++++++++++ .../pglogicalstream/schema_resolver_test.go | 90 ++++++++++ 7 files changed, 496 insertions(+), 42 deletions(-) create mode 100644 internal/impl/postgresql/pglogicalstream/schema_resolver.go create mode 100644 internal/impl/postgresql/pglogicalstream/schema_resolver_test.go diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index b61792a444..f08856dee5 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -14,6 +14,8 @@ import ( "encoding/json" "errors" "fmt" + "strconv" + "strings" "time" "github.com/Jeffail/checkpoint" @@ -81,6 +83,7 @@ Additionally, if ` + "`" + fieldStreamSnapshot + "`" + ` is set to true, then th This input adds the following metadata fields to each message: - table: Name of the table that the message originated from +- pg_schema: The PostgreSQL schema name that the table belongs to (e.g. "public", "tenant_foo"). Useful for per-schema routing when using schema patterns. - operation: Type of operation that generated the message: "read", "insert", "update", or "delete". "read" is from messages that are read in the initial snapshot phase. This will also be "begin" and "commit" if ` + "`" + fieldIncludeTxnMarkers + "`" + ` is enabled - lsn: the log sequence number in postgres - schema: The table schema in benthos common schema format, compatible with processors like parquet_encode @@ -105,8 +108,12 @@ This input adds the following metadata fields to each message: Example(10000). Default(1000)). Field(service.NewStringField(fieldSchema). - Description("The PostgreSQL schema from which to replicate data."). - Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`), + Description(`The PostgreSQL schema to replicate data from. Accepts an exact schema name or a glob pattern using ` + "`*`" + ` as a wildcard to match multiple schemas. + +When a pattern is used, all schemas whose names match the pattern are replicated using a single replication slot and publication. This is useful for multi-tenant databases where each tenant has its own schema (e.g. ` + "`tenant_*`" + ` matches ` + "`tenant_foo`" + `, ` + "`tenant_bar`" + `, etc.). + +Double-quoted identifiers are treated as exact names and do not support wildcards.`). + Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`, "tenant_*", "*"), ). Field(service.NewStringListField(fieldTables). Description("A list of table names to include in the logical replication. Each table should be specified as a separate item."). @@ -242,6 +249,15 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser if schema, err = conf.FieldString(fieldSchema); err != nil { return nil, err } + if err = validateSchemaPattern(schema); err != nil { + return nil, fmt.Errorf("invalid schema: %w", err) + } + // Normalize unquoted patterns to lower-case: PostgreSQL folds unquoted + // identifiers at creation time, so TENANT_* and tenant_* resolve identically. + // Normalizing early avoids silent case-folding surprises in resolveSchemas. + if !strings.HasPrefix(schema, `"`) { + schema = strings.ToLower(schema) + } if tables, err = conf.FieldStringList(fieldTables); err != nil { return nil, err @@ -321,8 +337,8 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser DBConfig: pgConnConfig, TLSConfig: pgConnConfig.TLSConfig, DBRawDSN: dsn, - DBSchema: schema, - DBTables: tables, + DBSchemaPattern: schema, + DBTables: tables, RefreshAuthToken: iamAuthTokenBuilder, IncludeTxnMarkers: includeTxnMarkers, @@ -361,6 +377,37 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser return conf.WrapBatchInputExtractTracingSpanMapping("postgres_cdc", r) } +// validateSchemaPattern validates a schema name or glob pattern. +// Accepts exact postgres identifiers (letters/digits/underscores) and glob +// patterns that additionally allow '*' as a wildcard character. +// Double-quoted identifiers (e.g. "MySchema") are accepted as exact names; +// wildcards are not allowed inside quotes. +func validateSchemaPattern(s string) error { + if s == "" { + return errors.New("schema cannot be empty") + } + if strings.HasPrefix(s, `"`) { + if !strings.HasSuffix(s, `"`) || len(s) < 2 { + return errors.New("unterminated quoted identifier in schema") + } + if strings.ContainsRune(s, '*') { + return errors.New("wildcard '*' is not allowed inside a quoted schema identifier") + } + return nil + } + for i, ch := range s { + if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '*' { + continue + } + return fmt.Errorf("invalid character %q at position %d in schema pattern %q", ch, i, s) + } + first := rune(s[0]) + if !(first == '_' || first == '*' || (first >= 'a' && first <= 'z') || (first >= 'A' && first <= 'Z')) { + return fmt.Errorf("schema pattern %q must start with a letter, underscore, or '*'", s) + } + return nil +} + // validateSimpleString ensures we aren't vuln to SQL injection. func validateSimpleString(s string) error { for _, b := range []byte(s) { @@ -475,6 +522,7 @@ func (p *pgStreamInput) processStream(pgStream *pglogicalstream.Stream, batcher } batchMsg := service.NewMessage(mb) batchMsg.MetaSet("table", msg.Table) + batchMsg.MetaSet("pg_schema", msg.Schema) batchMsg.MetaSet("operation", string(msg.Operation)) if msg.LSN != nil { batchMsg.MetaSet("lsn", *msg.LSN) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index c3115986e3..31eb6b8a05 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1052,13 +1052,20 @@ postgres_cdc: _, err = db.Exec(`INSERT INTO "FlightsCompositePK" ("Seq", "Name", "CreatedAt") VALUES ($1, $2, $3);`, 2, "bravo", "2006-01-02T15:04:05Z07:00") require.NoError(t, err) - _, err = db.Exec(`INSERT INTO flights (name, created_at) VALUES ($1, $2);`, "bravo", "2006-01-02T15:04:05Z07:00") + var flightsID int + err = db.QueryRow(`INSERT INTO flights (name, created_at) VALUES ($1, $2) RETURNING id;`, "bravo", "2006-01-02T15:04:05Z07:00").Scan(&flightsID) + require.NoError(t, err) + + _, err = db.Exec(`UPDATE flights SET name = $1 WHERE id = $2;`, "charlie", flightsID) + require.NoError(t, err) + + _, err = db.Exec(`DELETE FROM flights WHERE id = $1;`, flightsID) require.NoError(t, err) assert.EventuallyWithT(t, func(c *assert.CollectT) { outBatchMut.Lock() defer outBatchMut.Unlock() - assert.Len(c, outBatches, 4, "got: %#v", outBatches) + assert.Len(c, outBatches, 6, "got: %#v", outBatches) }, time.Second*25, time.Millisecond*100) require.ElementsMatch( @@ -1068,20 +1075,42 @@ postgres_cdc: map[string]any{ "operation": "read", "table": "FlightsCompositePK", + "pg_schema": "public", }, map[string]any{ "operation": "read", "table": "flights", + "pg_schema": "public", }, map[string]any{ - "operation": "insert", - "table": "flights", - "lsn": "XXX/XXX", + "operation": "insert", + "table": "FlightsCompositePK", + "lsn": "XXX/XXX", + "commit_ts_ms": "SET", + "pg_schema": "public", }, map[string]any{ - "operation": "insert", - "table": "FlightsCompositePK", - "lsn": "XXX/XXX", + "operation": "insert", + "table": "flights", + "lsn": "XXX/XXX", + "commit_ts_ms": "SET", + "pg_schema": "public", + }, + map[string]any{ + "operation": "update", + "table": "flights", + "lsn": "XXX/XXX", + "commit_ts_ms": "SET", + "before": "SET", + "pg_schema": "public", + }, + map[string]any{ + "operation": "delete", + "table": "flights", + "lsn": "XXX/XXX", + "commit_ts_ms": "SET", + "before": "SET", + "pg_schema": "public", }, }, ) @@ -1429,3 +1458,124 @@ postgres_cdc: } assert.Equal(t, "STRING", byName["extra"], "new 'extra' column should have type STRING") } + +func TestIntegrationMultiSchemaSnapshotAndCDC(t *testing.T) { + integration.CheckSkip(t) + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + // Two tenant schemas with the same table name, replicated on a single slot. + for _, schema := range []string{"tenant_a", "tenant_b"} { + _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", schema)) + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf( + "CREATE TABLE %s.events (id SERIAL PRIMARY KEY, name TEXT)", schema)) + require.NoError(t, err) + } + + // Pre-load snapshot data: 2 rows in tenant_a, 1 in tenant_b. + _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('alice'), ('bob')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_b.events (name) VALUES ('carol')") + require.NoError(t, err) + + type msgMeta struct { + pgSchema string + table string + operation string + lsn string + } + + var ( + mu sync.Mutex + collected []msgMeta + ) + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: multi_schema_test_slot + stream_snapshot: true + schema: tenant_* + tables: + - events +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + m := msgMeta{} + m.pgSchema, _ = msg.MetaGet("pg_schema") + m.table, _ = msg.MetaGet("table") + m.operation, _ = msg.MetaGet("operation") + m.lsn, _ = msg.MetaGet("lsn") + collected = append(collected, m) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { _ = stream.Run(t.Context()) }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + // Wait for all 3 snapshot rows. + assert.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(collected) >= 3 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for snapshot rows") + + // Insert CDC rows. + _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('dave')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_b.events (name) VALUES ('eve')") + require.NoError(t, err) + + // Wait for 2 CDC rows (total 5). + assert.EventuallyWithT(t, func(c *assert.CollectT) { + mu.Lock() + defer mu.Unlock() + assert.Len(c, collected, 5) + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for CDC rows") + + mu.Lock() + defer mu.Unlock() + + var snapshots, cdcMsgs []msgMeta + for _, m := range collected { + if m.operation == "read" { + snapshots = append(snapshots, m) + } else { + cdcMsgs = append(cdcMsgs, m) + } + } + + // Snapshot assertions. + require.Len(t, snapshots, 3) + snapshotSchemas := make(map[string]int) + for _, m := range snapshots { + assert.Equal(t, "events", m.table, "snapshot: table should be bare name without schema prefix") + assert.Empty(t, m.lsn, "snapshot rows have no LSN") + snapshotSchemas[m.pgSchema]++ + } + assert.Equal(t, 2, snapshotSchemas["tenant_a"], "expected 2 snapshot rows from tenant_a") + assert.Equal(t, 1, snapshotSchemas["tenant_b"], "expected 1 snapshot row from tenant_b") + + // CDC assertions. + require.Len(t, cdcMsgs, 2) + cdcSchemas := make(map[string]int) + for _, m := range cdcMsgs { + assert.Equal(t, "insert", m.operation) + assert.Equal(t, "events", m.table) + assert.NotEmpty(t, m.lsn, "CDC rows must have an LSN") + cdcSchemas[m.pgSchema]++ + } + assert.Equal(t, 1, cdcSchemas["tenant_a"], "expected 1 CDC row from tenant_a") + assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") +} diff --git a/internal/impl/postgresql/pglogicalstream/config.go b/internal/impl/postgresql/pglogicalstream/config.go index 73ff4c1c4d..8fed67862e 100644 --- a/internal/impl/postgresql/pglogicalstream/config.go +++ b/internal/impl/postgresql/pglogicalstream/config.go @@ -24,8 +24,10 @@ type Config struct { DBConfig *pgconn.Config DBRawDSN string TLSConfig *tls.Config - DBSchema string - DBTables []string + // DBSchemaPattern is the schema to replicate from. Accepts an exact schema + // name or a glob pattern using '*' as a wildcard (e.g. "tenant_*", "*"). + DBSchemaPattern string + DBTables []string // Refreshes short lived IAM auth token that is treated as a password RefreshAuthToken func(ctx context.Context) error // ReplicationSlotName is the name of the replication slot to use diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 5ebce38064..092669cba8 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -91,18 +91,29 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { return nil, err } - schema, err := sanitize.NormalizePostgresIdentifier(config.DBSchema) + schemas, err := resolveSchemas(ctx, dbConn, config.DBSchemaPattern) if err != nil { - return nil, fmt.Errorf("invalid schema name %q: %w", config.DBSchema, err) + return nil, fmt.Errorf("resolving schema pattern %q: %w", config.DBSchemaPattern, err) } + if len(schemas) == 0 { + return nil, fmt.Errorf("no schemas found matching pattern %q", config.DBSchemaPattern) + } + config.Logger.Infof("Schema pattern %q resolved to %d schema(s): %v", config.DBSchemaPattern, len(schemas), schemas) - tables := []TableFQN{} + normalizedTables := make([]string, 0, len(config.DBTables)) for _, table := range config.DBTables { normalized, err := sanitize.NormalizePostgresIdentifier(table) if err != nil { return nil, fmt.Errorf("invalid table name %q: %w", table, err) } - tables = append(tables, TableFQN{Schema: schema, Table: normalized}) + normalizedTables = append(normalizedTables, normalized) + } + + tables := make([]TableFQN, 0, len(schemas)*len(normalizedTables)) + for _, schema := range schemas { + for _, table := range normalizedTables { + tables = append(tables, TableFQN{Schema: schema, Table: table}) + } } batchSize := 1000 if config.BatchSize > 0 { diff --git a/internal/impl/postgresql/pglogicalstream/pglogrepl.go b/internal/impl/postgresql/pglogicalstream/pglogrepl.go index f92e222d0d..df2f80c933 100644 --- a/internal/impl/postgresql/pglogicalstream/pglogrepl.go +++ b/internal/impl/postgresql/pglogicalstream/pglogrepl.go @@ -23,7 +23,6 @@ import ( "encoding/binary" "errors" "fmt" - "slices" "strconv" "strings" "time" @@ -335,41 +334,66 @@ func CreatePublication(ctx context.Context, conn *pgconn.PgConn, publicationName return nil } - tablesToRemoveFromPublication := []TableFQN{} - tablesToAddToPublication := []TableFQN{} - for _, table := range tables { - if !slices.Contains(pubTables, table) { - tablesToAddToPublication = append(tablesToAddToPublication, table) - } + // Build sets for O(1) lookup — avoids O(n²) slices.Contains when reconciling + // large publication table lists (e.g. 100 schemas × 5 tables = 500 entries). + wantSet := make(map[TableFQN]struct{}, len(tables)) + for _, t := range tables { + wantSet[t] = struct{}{} + } + haveSet := make(map[TableFQN]struct{}, len(pubTables)) + for _, t := range pubTables { + haveSet[t] = struct{}{} } - for _, table := range pubTables { - if !slices.Contains(tables, table) { - tablesToRemoveFromPublication = append(tablesToRemoveFromPublication, table) + var tablesToAdd, tablesToRemove []TableFQN + for _, t := range tables { + if _, ok := haveSet[t]; !ok { + tablesToAdd = append(tablesToAdd, t) + } + } + for _, t := range pubTables { + if _, ok := wantSet[t]; !ok { + tablesToRemove = append(tablesToRemove, t) } } - // remove tables from publication - for _, dropTable := range tablesToRemoveFromPublication { - sq, err := sanitize.SQLQuery(fmt.Sprintf(`ALTER PUBLICATION %s DROP TABLE %s;`, publicationName, dropTable.String())) + // Batch DROP: single ALTER statement for all removed tables. + if len(tablesToRemove) > 0 { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("ALTER PUBLICATION %s DROP TABLE ", publicationName)) + for i, t := range tablesToRemove { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(t.String()) + } + sb.WriteByte(';') + sq, err := sanitize.SQLQuery(sb.String()) if err != nil { - return fmt.Errorf("sanitizing drop table query: %w", err) + return fmt.Errorf("sanitizing drop tables query: %w", err) } - result = conn.Exec(ctx, sq) - if _, err := result.ReadAll(); err != nil { - return fmt.Errorf("removing table from publication: %w", err) + if _, err := conn.Exec(ctx, sq).ReadAll(); err != nil { + return fmt.Errorf("removing tables from publication: %w", err) } } - // add tables to publication - for _, addTable := range tablesToAddToPublication { - sq, err := sanitize.SQLQuery(fmt.Sprintf("ALTER PUBLICATION %s ADD TABLE %s;", publicationName, addTable.String())) + // Batch ADD: single ALTER statement for all new tables. + if len(tablesToAdd) > 0 { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("ALTER PUBLICATION %s ADD TABLE ", publicationName)) + for i, t := range tablesToAdd { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(t.String()) + } + sb.WriteByte(';') + sq, err := sanitize.SQLQuery(sb.String()) if err != nil { - return fmt.Errorf("sanitizing add table query: %w", err) + return fmt.Errorf("sanitizing add tables query: %w", err) } - result = conn.Exec(ctx, sq) - if _, err := result.ReadAll(); err != nil { - return fmt.Errorf("adding table to publication: %w", err) + if _, err := conn.Exec(ctx, sq).ReadAll(); err != nil { + return fmt.Errorf("adding tables to publication: %w", err) } } diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go new file mode 100644 index 0000000000..8af4f7712d --- /dev/null +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -0,0 +1,129 @@ +// Copyright 2024 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md + +package pglogicalstream + +import ( + "context" + "fmt" + "strings" + + "github.com/jackc/pgx/v5/pgconn" + + "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/sanitize" +) + +// resolveSchemas expands a schema name or glob pattern into the set of +// quoted PostgreSQL identifiers that exist in the database. +// +// For unquoted patterns (e.g. "tenant_*") the pattern is matched +// case-insensitively against information_schema.schemata using LIKE, because +// PostgreSQL folds unquoted identifiers to lower-case at creation time. +// +// For quoted identifiers (e.g. `"MySchema"`) an exact case-sensitive lookup +// is performed. +// +// System schemas (pg_* and information_schema) are always excluded so that +// wildcard patterns like "*" do not attempt to replicate catalog tables. +// +// Returns an error if the query fails or if no schemas match the pattern. +// schemaPatternToLike converts a schema name or glob pattern into the LIKE +// pattern used by resolveSchemas. Extracted for unit testing. +// +// For quoted identifiers the inner name is exact-escaped (no wildcard expansion). +// For unquoted patterns the '*' wildcard is converted to '%' and the input is +// folded to lower-case to match PostgreSQL's identifier folding. +func schemaPatternToLike(pattern string) (string, error) { + if strings.HasPrefix(pattern, `"`) { + unquoted, err := sanitize.UnquotePostgresIdentifier(pattern) + if err != nil { + return "", fmt.Errorf("invalid quoted schema identifier %q: %w", pattern, err) + } + return escapeLike(unquoted), nil + } + return globToLike(strings.ToLower(pattern)), nil +} + +func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) ([]string, error) { + likePattern, err := schemaPatternToLike(pattern) + if err != nil { + return nil, err + } + + q, err := sanitize.SQLQuery( + "SELECT schema_name FROM information_schema.schemata WHERE schema_name LIKE $1 ESCAPE '!' AND schema_name NOT LIKE 'pg!_%' ESCAPE '!' AND schema_name != 'information_schema'", + likePattern, + ) + if err != nil { + return nil, fmt.Errorf("building schema resolution query: %w", err) + } + + results, err := conn.Exec(ctx, q).ReadAll() + if err != nil { + return nil, fmt.Errorf("querying schemas matching %q: %w", pattern, err) + } + + var schemas []string + if len(results) > 0 { + for _, row := range results[0].Rows { + // QuotePostgresIdentifier preserves the exact stored name (including + // case for case-sensitive schemas), unlike NormalizePostgresIdentifier + // which would incorrectly fold to lower-case. + schemas = append(schemas, sanitize.QuotePostgresIdentifier(string(row[0]))) + } + } + return schemas, nil +} + +// globToLike converts an unquoted glob pattern (using '*' as wildcard) into a +// PostgreSQL LIKE pattern that uses '!' as the escape character. +// +// Mapping: +// - '*' → '%' (zero or more characters) +// - '_' → '!_' (literal underscore, not the LIKE single-char wildcard) +// - '%' → '!%' (literal percent, not the LIKE multi-char wildcard) +// - '!' → '!!' (literal escape character) +func globToLike(pattern string) string { + var b strings.Builder + b.Grow(len(pattern) + 4) + for _, ch := range pattern { + switch ch { + case '*': + b.WriteByte('%') + case '_': + b.WriteString("!_") + case '%': + b.WriteString("!%") + case '!': + b.WriteString("!!") + default: + b.WriteRune(ch) + } + } + return b.String() +} + +// escapeLike escapes LIKE metacharacters in s without expanding any wildcards. +// Used for exact quoted-identifier lookups. +func escapeLike(s string) string { + var b strings.Builder + b.Grow(len(s)) + for _, ch := range s { + switch ch { + case '_': + b.WriteString("!_") + case '%': + b.WriteString("!%") + case '!': + b.WriteString("!!") + default: + b.WriteRune(ch) + } + } + return b.String() +} diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go b/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go new file mode 100644 index 0000000000..7ee213f6e4 --- /dev/null +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go @@ -0,0 +1,90 @@ +// Copyright 2024 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md + +package pglogicalstream + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGlobToLike(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"public", "public"}, + {"tenant_*", "tenant!_%"}, + {"*", "%"}, + {"tenant_a", "tenant!_a"}, + {"100%", "100!%"}, + {"a!b", "a!!b"}, + {"multi_*_end", "multi!_%!_end"}, + {"", ""}, + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + assert.Equal(t, tt.expected, globToLike(tt.input)) + }) + } +} + +func TestSchemaPatternToLike(t *testing.T) { + tests := []struct { + pattern string + expected string + errContains string + }{ + // Unquoted glob patterns — folded to lower-case, '*' → '%', '_' escaped. + {"public", "public", ""}, + {"tenant_*", "tenant!_%", ""}, + {"*", "%", ""}, + {"schema_1", "schema!_1", ""}, + // Upper-case is folded: TENANT_* matches the same rows as tenant_*. + {"TENANT_*", "tenant!_%", ""}, + // Quoted exact identifier — case preserved, no wildcard expansion. + {`"MySchema"`, "MySchema", ""}, + {`"schema_1"`, "schema!_1", ""}, + {`"has%bang!"`, "has!%bang!!", ""}, + // Unterminated quoted identifier → error. + {`"bad`, "", "invalid quoted schema identifier"}, + } + for _, tt := range tests { + t.Run(tt.pattern, func(t *testing.T) { + got, err := schemaPatternToLike(tt.pattern) + if tt.errContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errContains) + return + } + require.NoError(t, err) + assert.Equal(t, tt.expected, got) + }) + } +} + +func TestEscapeLike(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"MySchema", "MySchema"}, + {"schema_1", "schema!_1"}, + {"100%", "100!%"}, + {"bang!bang", "bang!!bang"}, + {"has_a%b!c", "has!_a!%b!!c"}, + {"", ""}, + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + assert.Equal(t, tt.expected, escapeLike(tt.input)) + }) + } +} From c38fd7a709654da2c871c1acd994ab8084a71be8 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Mon, 6 Jul 2026 22:27:03 +0300 Subject: [PATCH 02/48] postgres_cdc: reject empty quoted schema identifier and fix misleading godoc --- internal/impl/postgresql/input_pg_stream.go | 2 +- internal/impl/postgresql/pglogicalstream/schema_resolver.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index f08856dee5..733155a281 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -387,7 +387,7 @@ func validateSchemaPattern(s string) error { return errors.New("schema cannot be empty") } if strings.HasPrefix(s, `"`) { - if !strings.HasSuffix(s, `"`) || len(s) < 2 { + if !strings.HasSuffix(s, `"`) || len(s) < 3 { return errors.New("unterminated quoted identifier in schema") } if strings.ContainsRune(s, '*') { diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 8af4f7712d..8920121e99 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -31,7 +31,9 @@ import ( // System schemas (pg_* and information_schema) are always excluded so that // wildcard patterns like "*" do not attempt to replicate catalog tables. // -// Returns an error if the query fails or if no schemas match the pattern. +// Returns an error if the query fails. Returns (nil, nil) if no schemas match. +// The caller is responsible for treating an empty result as an error. + // schemaPatternToLike converts a schema name or glob pattern into the LIKE // pattern used by resolveSchemas. Extracted for unit testing. // From bda218d4b793dd1a8e91841f3f9172e4521dfd5b Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Tue, 7 Jul 2026 17:54:33 +0300 Subject: [PATCH 03/48] postgres_cdc: add tests/current/ Docker Compose + Taskfile manual test harness Two-schema (tenant_a, tenant_b) Postgres 16 setup that exercises the multi-schema CDC pipeline end-to-end. Also adds schema_validation unit tests that verify invalid patterns are rejected at startup without a DB. --- .../postgresql/tests/current/Taskfile.yaml | 98 +++++++++++++++++++ .../tests/current/docker-compose.yaml | 36 +++++++ .../impl/postgresql/tests/current/setup.sql | 33 +++++++ .../postgresql/tests/current/test_config.yaml | 41 ++++++++ 4 files changed, 208 insertions(+) create mode 100644 internal/impl/postgresql/tests/current/Taskfile.yaml create mode 100644 internal/impl/postgresql/tests/current/docker-compose.yaml create mode 100644 internal/impl/postgresql/tests/current/setup.sql create mode 100644 internal/impl/postgresql/tests/current/test_config.yaml diff --git a/internal/impl/postgresql/tests/current/Taskfile.yaml b/internal/impl/postgresql/tests/current/Taskfile.yaml new file mode 100644 index 0000000000..75551c4d34 --- /dev/null +++ b/internal/impl/postgresql/tests/current/Taskfile.yaml @@ -0,0 +1,98 @@ +version: "3" + +vars: + PG_DSN: '{{.PG_DSN | default "postgres://postgres:postgres@localhost:5433/testdb?sslmode=disable"}}' + +tasks: + # ── Infrastructure ──────────────────────────────────────────────────────────── + + up: + desc: Start PostgreSQL and run schema/data setup + cmds: + - docker compose up -d postgres + - docker compose run --rm setup + + down: + desc: Stop and remove all containers and volumes + cmds: + - docker compose down -v + + reset: + desc: Full teardown + bring back up (also drops and recreates the replication slot) + cmds: + - task: down + - task: up + + # ── Slot management ─────────────────────────────────────────────────────────── + + slot:drop: + desc: Drop the replication slot so the test can be re-run from scratch + cmds: + - | + psql "{{.PG_DSN}}" -c \ + "SELECT pg_drop_replication_slot('multi_schema_test_slot') + FROM pg_replication_slots + WHERE slot_name = 'multi_schema_test_slot';" + + # ── Run the pipeline ────────────────────────────────────────────────────────── + + run: + desc: Run the multi-schema CDC pipeline (streams snapshot then live CDC) + env: + PG_DSN: '{{.PG_DSN}}' + cmds: + - task: slot:drop + - go run ../../../../../cmd/redpanda-connect/main.go run ./test_config.yaml + + # ── Test data ───────────────────────────────────────────────────────────────── + + data:insert: + desc: Insert CDC rows into both tenant schemas (triggers insert events) + cmds: + - psql "{{.PG_DSN}}" -c "INSERT INTO tenant_a.events (name) VALUES ('dave'), ('eve');" + - psql "{{.PG_DSN}}" -c "INSERT INTO tenant_b.events (name) VALUES ('frank');" + + data:update: + desc: Update a row in tenant_a (triggers update event with 'before' field) + cmds: + - psql "{{.PG_DSN}}" -c "UPDATE tenant_a.events SET status = 'updated' WHERE name = 'dave';" + + data:delete: + desc: Delete a row from tenant_b (triggers delete event with 'before' field) + cmds: + - psql "{{.PG_DSN}}" -c "DELETE FROM tenant_b.events WHERE name = 'frank';" + + data:all: + desc: Run all test data mutations in sequence (insert → update → delete) + cmds: + - task: data:insert + - task: data:update + - task: data:delete + + # ── Schema validation smoke test ────────────────────────────────────────────── + + test:invalid-schema: + desc: Confirm that an empty quoted schema identifier is rejected at startup + env: + PG_DSN: '{{.PG_DSN}}' + cmds: + - | + set +e + go run ../../../../../cmd/redpanda-connect/main.go run \ + --set 'input.postgres_cdc.schema=""' \ + ./test_config.yaml 2>&1 | head -5 + echo "exit $?" + # Expects: "invalid schema" error printed, process exits non-zero. + + # ── Quick sanity ────────────────────────────────────────────────────────────── + + psql: + desc: Open a psql shell to the test database + cmds: + - psql "{{.PG_DSN}}" + + status: + desc: Show active replication slots and publications + cmds: + - psql "{{.PG_DSN}}" -c "SELECT slot_name, active FROM pg_replication_slots;" + - psql "{{.PG_DSN}}" -c "SELECT pubname FROM pg_publication;" diff --git a/internal/impl/postgresql/tests/current/docker-compose.yaml b/internal/impl/postgresql/tests/current/docker-compose.yaml new file mode 100644 index 0000000000..896a807027 --- /dev/null +++ b/internal/impl/postgresql/tests/current/docker-compose.yaml @@ -0,0 +1,36 @@ +services: + postgres: + image: postgres:16 + container_name: pgtest-postgres + ports: + - "5433:5432" + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: testdb + # Enable logical replication — required for postgres_cdc + command: postgres -c wal_level=logical -c max_replication_slots=10 -c max_wal_senders=10 + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 3s + timeout: 5s + retries: 10 + + # One-shot setup container: creates schemas, tables, and seed data, then exits. + setup: + image: postgres:16 + container_name: pgtest-setup + depends_on: + postgres: + condition: service_healthy + environment: + PGPASSWORD: postgres + volumes: + - ./setup.sql:/setup.sql:ro + entrypoint: /bin/sh + command: + - -c + - | + psql -h postgres -U postgres -d testdb -f /setup.sql + echo "setup complete" + restart: "no" diff --git a/internal/impl/postgresql/tests/current/setup.sql b/internal/impl/postgresql/tests/current/setup.sql new file mode 100644 index 0000000000..ee40a376cc --- /dev/null +++ b/internal/impl/postgresql/tests/current/setup.sql @@ -0,0 +1,33 @@ +-- Multi-schema CDC test setup +-- Tests: schema glob (tenant_*), pg_schema metadata, commit_ts_ms, before (update/delete) + +-- ── Tenant schemas ──────────────────────────────────────────────────────────── + +CREATE SCHEMA IF NOT EXISTS tenant_a; +CREATE SCHEMA IF NOT EXISTS tenant_b; + +-- ── Events table (same shape in each schema) ────────────────────────────────── + +CREATE TABLE IF NOT EXISTS tenant_a.events ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS tenant_b.events ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- REPLICA IDENTITY FULL so update/delete messages carry the full before-row. +ALTER TABLE tenant_a.events REPLICA IDENTITY FULL; +ALTER TABLE tenant_b.events REPLICA IDENTITY FULL; + +-- ── Seed snapshot rows ──────────────────────────────────────────────────────── +-- These are visible during the initial snapshot (stream_snapshot: true). + +INSERT INTO tenant_a.events (name) VALUES ('alice'), ('bob'); +INSERT INTO tenant_b.events (name) VALUES ('carol'); diff --git a/internal/impl/postgresql/tests/current/test_config.yaml b/internal/impl/postgresql/tests/current/test_config.yaml new file mode 100644 index 0000000000..4a994e30c2 --- /dev/null +++ b/internal/impl/postgresql/tests/current/test_config.yaml @@ -0,0 +1,41 @@ +input: + postgres_cdc: + dsn: ${PG_DSN:postgres://postgres:postgres@localhost:5433/testdb?sslmode=disable} + slot_name: multi_schema_test_slot + stream_snapshot: true + # Glob pattern: replicates both tenant_a and tenant_b via one slot. + schema: tenant_* + tables: + - events + +pipeline: + processors: + # Annotate each message with all new metadata fields so the output clearly + # shows what the feature delivers. + - mapping: | + let op = @operation + let tbl = @table + let schema = @pg_schema + let lsn = @lsn + let ts_ms = @commit_ts_ms + let before = @before + + root = { + "operation": $op, + "pg_schema": $schema, + "table": $tbl, + "payload": this, + "lsn": if $lsn != null { $lsn } else { null }, + "commit_ts_ms": if $ts_ms != null { $ts_ms } else { null }, + "before": if $before != null { $before.string().parse_json() } else { null }, + } + +output: + stdout: + codec: lines + +logger: + level: INFO + +metrics: + none: {} From df2dca6dff49c4f9c50ca795d89815a7d107859a Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Tue, 7 Jul 2026 17:55:08 +0300 Subject: [PATCH 04/48] postgres_cdc: add commit_ts_ms and before metadata fields --- internal/impl/postgresql/input_pg_stream.go | 8 + internal/impl/postgresql/integration_test.go | 6 + .../pglogicalstream/logical_stream.go | 14 ++ .../replication_message_decoders.go | 23 +++ .../pglogicalstream/stream_message.go | 6 + .../tests/schema_validation/validate_test.go | 147 ++++++++++++++++++ 6 files changed, 204 insertions(+) create mode 100644 internal/impl/postgresql/tests/schema_validation/validate_test.go diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 733155a281..7acbe0dd62 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -530,6 +530,14 @@ func (p *pgStreamInput) processStream(pgStream *pglogicalstream.Stream, batcher if msg.ColumnSchema != nil { batchMsg.MetaSetImmut("schema", service.ImmutableAny{V: msg.ColumnSchema}) } + if msg.CommitTs != nil { + batchMsg.MetaSet("commit_ts_ms", strconv.FormatInt(msg.CommitTs.UnixMilli(), 10)) + } + if msg.Before != nil { + if beforeBytes, err := json.Marshal(msg.Before); err == nil { + batchMsg.MetaSet("before", string(beforeBytes)) + } + } if batcher.Add(batchMsg) { flush = true } diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 31eb6b8a05..6087b62a96 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1029,6 +1029,12 @@ postgres_cdc: if _, ok := d["lsn"]; ok { d["lsn"] = "XXX/XXX" // Consistent LSN for assertions below } + if _, ok := d["commit_ts_ms"]; ok { + d["commit_ts_ms"] = "SET" + } + if _, ok := d["before"]; ok { + d["before"] = "SET" + } delete(d, "schema") // Schema metadata tested separately in TestIntegrationPostgresCDCSchemaMetadata outBatches = append(outBatches, data) } diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 092669cba8..bd231a30f2 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -55,6 +55,7 @@ type Stream struct { heartbeat *heartbeat maxSnapshotWorkers int unchangedToastValue any + currentTxCommitTime *time.Time } // NewPgStream creates a new instance of the Stream struct. @@ -517,6 +518,14 @@ func (s *Stream) processChange(ctx context.Context, msgLSN LSN, xld XLogData, re delete(schemaCache, rel.RelationID) } + // Track commit timestamp: set on BEGIN (available for DML), clear on COMMIT. + if begin, ok := logicalMsg.(*BeginMessage); ok { + t := begin.CommitTime + s.currentTxCommitTime = &t + } else if _, ok := logicalMsg.(*CommitMessage); ok { + s.currentTxCommitTime = nil + } + // parse changes inside the transaction message, err := toStreamMessage(logicalMsg, relations, typeMap, s.unchangedToastValue) if err != nil { @@ -562,6 +571,11 @@ func (s *Stream) processChange(ctx context.Context, msgLSN LSN, xld XLogData, re } } + switch message.Operation { + case InsertOpType, UpdateOpType, DeleteOpType: + message.CommitTs = s.currentTxCommitTime + } + lsn := msgLSN.String() message.LSN = &lsn select { diff --git a/internal/impl/postgresql/pglogicalstream/replication_message_decoders.go b/internal/impl/postgresql/pglogicalstream/replication_message_decoders.go index 02466e85ed..1f93398c15 100644 --- a/internal/impl/postgresql/pglogicalstream/replication_message_decoders.go +++ b/internal/impl/postgresql/pglogicalstream/replication_message_decoders.go @@ -133,6 +133,28 @@ func toStreamMessage(logicalMsg Message, relations map[uint32]*RelationMessage, } } message.Data = values + if logicalMsg.OldTuple != nil { + before := map[string]any{} + for idx, col := range logicalMsg.OldTuple.Columns { + if idx >= len(rel.Columns) { + break + } + colName := rel.Columns[idx].Name + switch col.DataType { + case 'n': + before[colName] = nil + case 'u': + before[colName] = unchangedToastValue + case 't': + val, err := decodeTextColumnData(typeMap, col.Data, rel.Columns[idx].DataType, rel.Columns[idx].TypeModifier) + if err != nil { + return nil, fmt.Errorf("unable to decode before column data: %w", err) + } + before[colName] = val + } + } + message.Before = before + } case *DeleteMessage: rel, ok := relations[logicalMsg.RelationID] if !ok { @@ -159,6 +181,7 @@ func toStreamMessage(logicalMsg Message, relations map[uint32]*RelationMessage, } } message.Data = values + message.Before = values case *TruncateMessage: case *TypeMessage: case *OriginMessage: diff --git a/internal/impl/postgresql/pglogicalstream/stream_message.go b/internal/impl/postgresql/pglogicalstream/stream_message.go index 73e6795fee..0fe927f81f 100644 --- a/internal/impl/postgresql/pglogicalstream/stream_message.go +++ b/internal/impl/postgresql/pglogicalstream/stream_message.go @@ -8,6 +8,8 @@ package pglogicalstream +import "time" + // StreamMode represents the mode of the stream at the time of the message type StreamMode string @@ -47,4 +49,8 @@ type StreamMessage struct { // ColumnSchema contains the table's column schema in benthos common schema format. // It is set as message metadata and excluded from JSON serialization. ColumnSchema any `json:"-"` + // CommitTs is the commit timestamp of the enclosing transaction. Nil for snapshot reads. + CommitTs *time.Time `json:"-"` + // Before holds the pre-change row state for update and delete operations. Nil otherwise. + Before any `json:"-"` } diff --git a/internal/impl/postgresql/tests/schema_validation/validate_test.go b/internal/impl/postgresql/tests/schema_validation/validate_test.go new file mode 100644 index 0000000000..a2acdc2e42 --- /dev/null +++ b/internal/impl/postgresql/tests/schema_validation/validate_test.go @@ -0,0 +1,147 @@ +// Copyright 2024 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md + +// Package schema_validation tests postgres_cdc schema pattern validation +// through the public service config API. No database connection is required: +// invalid schemas are rejected during stream construction (before any network +// I/O), so stream.Run returns synchronously for the invalid-schema cases. +package schema_validation_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + _ "github.com/redpanda-data/benthos/v4/public/components/pure" // registers none tracer and other built-ins + + "github.com/redpanda-data/connect/v4/internal/license" + _ "github.com/redpanda-data/connect/v4/internal/impl/postgresql" // registers postgres_cdc +) + +// postgresStream builds a postgres_cdc stream with the given YAML schema value +// (the raw fragment that appears after "schema: " in YAML) and injects a test +// enterprise license. Returns the stream ready to run. +func postgresStream(t *testing.T, schemaYAML string) *service.Stream { + t.Helper() + yaml := fmt.Sprintf(` +postgres_cdc: + dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable + schema: %s + slot_name: test_slot + tables: + - events +`, schemaYAML) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: ERROR`)) + require.NoError(t, sb.AddInputYAML(yaml)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, _ service.MessageBatch) error { + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + return stream +} + +// TestInvalidSchemaPatterns verifies that invalid schema values are rejected +// during stream construction — before any database connection is attempted. +// stream.Run returns synchronously (no goroutine needed) when the constructor +// fails. +func TestInvalidSchemaPatterns(t *testing.T) { + tests := []struct { + name string + schemaYAML string + wantErrMsg string + }{ + { + // Regression test: len("") == 2 used to pass the old `len(s) < 2` + // guard. Fixed to `len(s) < 3`. + name: "empty quoted identifier", + schemaYAML: `'""'`, + wantErrMsg: "invalid schema", + }, + { + name: "digit-first unquoted pattern", + schemaYAML: `"1abc"`, + wantErrMsg: "invalid schema", + }, + { + name: "unterminated quoted identifier", + schemaYAML: `'"unclosed'`, + wantErrMsg: "invalid schema", + }, + { + name: "hyphen in unquoted pattern", + schemaYAML: `schema-name`, + wantErrMsg: "invalid schema", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stream := postgresStream(t, tt.schemaYAML) + + // Run returns synchronously when the input constructor fails — + // no timeout context needed, but use one as a safety net. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err := stream.Run(ctx) + require.Error(t, err, "expected stream construction to fail") + assert.Contains(t, err.Error(), tt.wantErrMsg, + "error should indicate schema validation failure") + }) + } +} + +// TestValidSchemaPatterns verifies that valid schema values pass construction +// and are only rejected later (at DB-connect time). We run the stream briefly +// and confirm no "invalid schema" error surfaces. +func TestValidSchemaPatterns(t *testing.T) { + tests := []struct { + name string + schemaYAML string + }{ + {name: "exact unquoted schema", schemaYAML: `public`}, + {name: "glob pattern", schemaYAML: `tenant_*`}, + {name: "wildcard", schemaYAML: `"*"`}, + {name: "quoted exact identifier", schemaYAML: `'"MySchema"'`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stream := postgresStream(t, tt.schemaYAML) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + errCh := make(chan error, 1) + go func() { errCh <- stream.Run(ctx) }() + + select { + case err := <-errCh: + // Stream stopped before timeout — must not be a schema error. + if err != nil { + assert.NotContains(t, err.Error(), "invalid schema", + "valid schema %q should not trigger schema validation error", tt.schemaYAML) + } + case <-ctx.Done(): + // Stream is still running after timeout — constructor succeeded, + // stream is attempting DB connection. This is the expected path. + _ = stream.StopWithin(2 * time.Second) + } + }) + } +} From fcb0427d6a1a030a36adf5b963fddf3380565917 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Fri, 10 Jul 2026 12:55:07 +0300 Subject: [PATCH 05/48] postgres_cdc: fix lint and docs --- docs/modules/components/pages/inputs/postgres_cdc.adoc | 10 +++++++++- internal/impl/postgresql/input_pg_stream.go | 8 ++++---- internal/impl/postgresql/pglogicalstream/pglogrepl.go | 4 ++-- .../tests/schema_validation/validate_test.go | 4 ++-- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index a7d64a843b..e1c2257518 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -170,7 +170,11 @@ snapshot_batch_size: 10000 === `schema` -The PostgreSQL schema from which to replicate data. +The PostgreSQL schema to replicate data from. Accepts an exact schema name or a glob pattern using `*` as a wildcard to match multiple schemas. + +When a pattern is used, all schemas whose names match the pattern are replicated using a single replication slot and publication. This is useful for multi-tenant databases where each tenant has its own schema (e.g. `tenant_*` matches `tenant_foo`, `tenant_bar`, etc.). + +Double-quoted identifiers are treated as exact names and do not support wildcards. *Type*: `string` @@ -182,6 +186,10 @@ The PostgreSQL schema from which to replicate data. schema: public schema: '"MyCaseSensitiveSchemaNeedingQuotes"' + +schema: tenant_* + +schema: '*' ``` === `tables` diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index e6e1b509f8..dfec0b4881 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -110,9 +110,9 @@ This input adds the following metadata fields to each message: Example(10000). Default(1000)). Field(service.NewStringField(fieldSchema). - Description(`The PostgreSQL schema to replicate data from. Accepts an exact schema name or a glob pattern using ` + "`*`" + ` as a wildcard to match multiple schemas. + Description(`The PostgreSQL schema to replicate data from. Accepts an exact schema name or a glob pattern using `+"`*`"+` as a wildcard to match multiple schemas. -When a pattern is used, all schemas whose names match the pattern are replicated using a single replication slot and publication. This is useful for multi-tenant databases where each tenant has its own schema (e.g. ` + "`tenant_*`" + ` matches ` + "`tenant_foo`" + `, ` + "`tenant_bar`" + `, etc.). +When a pattern is used, all schemas whose names match the pattern are replicated using a single replication slot and publication. This is useful for multi-tenant databases where each tenant has its own schema (e.g. `+"`tenant_*`"+` matches `+"`tenant_foo`"+`, `+"`tenant_bar`"+`, etc.). Double-quoted identifiers are treated as exact names and do not support wildcards.`). Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`, "tenant_*", "*"), @@ -339,8 +339,8 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser DBConfig: pgConnConfig, TLSConfig: pgConnConfig.TLSConfig, DBRawDSN: dsn, - DBSchemaPattern: schema, - DBTables: tables, + DBSchemaPattern: schema, + DBTables: tables, RefreshAuthToken: iamAuthTokenBuilder, IncludeTxnMarkers: includeTxnMarkers, diff --git a/internal/impl/postgresql/pglogicalstream/pglogrepl.go b/internal/impl/postgresql/pglogicalstream/pglogrepl.go index df2f80c933..69385f753f 100644 --- a/internal/impl/postgresql/pglogicalstream/pglogrepl.go +++ b/internal/impl/postgresql/pglogicalstream/pglogrepl.go @@ -360,7 +360,7 @@ func CreatePublication(ctx context.Context, conn *pgconn.PgConn, publicationName // Batch DROP: single ALTER statement for all removed tables. if len(tablesToRemove) > 0 { var sb strings.Builder - sb.WriteString(fmt.Sprintf("ALTER PUBLICATION %s DROP TABLE ", publicationName)) + fmt.Fprintf(&sb, "ALTER PUBLICATION %s DROP TABLE ", publicationName) for i, t := range tablesToRemove { if i > 0 { sb.WriteString(", ") @@ -380,7 +380,7 @@ func CreatePublication(ctx context.Context, conn *pgconn.PgConn, publicationName // Batch ADD: single ALTER statement for all new tables. if len(tablesToAdd) > 0 { var sb strings.Builder - sb.WriteString(fmt.Sprintf("ALTER PUBLICATION %s ADD TABLE ", publicationName)) + fmt.Fprintf(&sb, "ALTER PUBLICATION %s ADD TABLE ", publicationName) for i, t := range tablesToAdd { if i > 0 { sb.WriteString(", ") diff --git a/internal/impl/postgresql/tests/schema_validation/validate_test.go b/internal/impl/postgresql/tests/schema_validation/validate_test.go index a2acdc2e42..e45e4bf2f2 100644 --- a/internal/impl/postgresql/tests/schema_validation/validate_test.go +++ b/internal/impl/postgresql/tests/schema_validation/validate_test.go @@ -21,11 +21,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/redpanda-data/benthos/v4/public/service" _ "github.com/redpanda-data/benthos/v4/public/components/pure" // registers none tracer and other built-ins + "github.com/redpanda-data/benthos/v4/public/service" - "github.com/redpanda-data/connect/v4/internal/license" _ "github.com/redpanda-data/connect/v4/internal/impl/postgresql" // registers postgres_cdc + "github.com/redpanda-data/connect/v4/internal/license" ) // postgresStream builds a postgres_cdc stream with the given YAML schema value From 7bcc3ac5af7cb21e4bbabd97e379443fd477810a Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Fri, 10 Jul 2026 13:21:46 +0300 Subject: [PATCH 06/48] postgres_cdc: review fixes and test coverage --- CHANGELOG.md | 4 +++ .../components/pages/inputs/postgres_cdc.adoc | 2 ++ internal/impl/postgresql/input_pg_stream.go | 9 +++-- internal/impl/postgresql/integration_test.go | 33 +++++++++++++++++++ 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77f8a2e534..93fa77403b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. ## Unreleased +### Added + +- postgres_cdc: Postgres CDC now accepts a glob pattern for the `schema` field (e.g. `tenant_*`), replicating all matching schemas through a single replication slot. Useful for multi-tenant databases where each tenant has its own schema. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) + ### Fixed - general: The CGO-enabled distribution binary now embeds the IANA time zone database via the `timetzdata` build tag, matching the other distributions, so `time.LoadLocation` works in minimal runtimes without system tzdata instead of silently falling back to UTC (which shifts JQL date predicates in the `jira` input). ([@squiidz](https://github.com/squiidz), [#4583](https://github.com/redpanda-data/connect/pull/4583)) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index e1c2257518..069c0f1bfe 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -176,6 +176,8 @@ When a pattern is used, all schemas whose names match the pattern are replicated Double-quoted identifiers are treated as exact names and do not support wildcards. +Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted. + *Type*: `string` diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index dfec0b4881..f8d0b0fde3 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -26,6 +26,7 @@ import ( "github.com/redpanda-data/connect/v4/internal/asyncroutine" "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream" + "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/sanitize" "github.com/redpanda-data/connect/v4/internal/license" ) @@ -114,7 +115,9 @@ This input adds the following metadata fields to each message: When a pattern is used, all schemas whose names match the pattern are replicated using a single replication slot and publication. This is useful for multi-tenant databases where each tenant has its own schema (e.g. `+"`tenant_*`"+` matches `+"`tenant_foo`"+`, `+"`tenant_bar`"+`, etc.). -Double-quoted identifiers are treated as exact names and do not support wildcards.`). +Double-quoted identifiers are treated as exact names and do not support wildcards. + +Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted.`). Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`, "tenant_*", "*"), ). Field(service.NewStringListField(fieldTables). @@ -389,8 +392,8 @@ func validateSchemaPattern(s string) error { return errors.New("schema cannot be empty") } if strings.HasPrefix(s, `"`) { - if !strings.HasSuffix(s, `"`) || len(s) < 3 { - return errors.New("unterminated quoted identifier in schema") + if _, err := sanitize.UnquotePostgresIdentifier(s); err != nil { + return fmt.Errorf("invalid quoted schema identifier: %w", err) } if strings.ContainsRune(s, '*') { return errors.New("wildcard '*' is not allowed inside a quoted schema identifier") diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index aeae22b0c7..30068b6d2c 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1600,3 +1600,36 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_a"], "expected 1 CDC row from tenant_a") assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } + +func TestIntegrationNoSchemasMatchedError(t *testing.T) { + integration.CheckSkip(t) + databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: no_schema_match_slot + schema: nonexistent_schema_zzz_* + tables: + - events +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: ERROR`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, _ service.MessageBatch) error { + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + err = stream.Run(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "no schemas found matching pattern") +} From d39de9a3e5976a9868ee97674ab54273da689001 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Mon, 13 Jul 2026 10:26:28 +0300 Subject: [PATCH 07/48] postgres_cdc: fix tests --- .../impl/postgresql/pglogicalstream/sanitize/sanitize.go | 2 +- internal/plugins/cdctest/cdc_conformance_test.go | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/sanitize/sanitize.go b/internal/impl/postgresql/pglogicalstream/sanitize/sanitize.go index febdb3311f..c3826b0037 100644 --- a/internal/impl/postgresql/pglogicalstream/sanitize/sanitize.go +++ b/internal/impl/postgresql/pglogicalstream/sanitize/sanitize.go @@ -384,7 +384,7 @@ func QuotePostgresIdentifier(name string) string { // UnquotePostgresIdentifier returns the valid unescaped identifier. func UnquotePostgresIdentifier(quoted string) (string, error) { var output strings.Builder - if !strings.HasPrefix(quoted, `"`) || !strings.HasSuffix(quoted, `"`) || len(quoted) < 2 { + if !strings.HasPrefix(quoted, `"`) || !strings.HasSuffix(quoted, `"`) || len(quoted) < 3 { return "", errors.New("missing quotes for identifier") } unquoted := quoted[1 : len(quoted)-1] diff --git a/internal/plugins/cdctest/cdc_conformance_test.go b/internal/plugins/cdctest/cdc_conformance_test.go index cd5eac4de5..31141ddc2a 100644 --- a/internal/plugins/cdctest/cdc_conformance_test.go +++ b/internal/plugins/cdctest/cdc_conformance_test.go @@ -92,13 +92,6 @@ var knownNonConformant = map[string]map[string]string{ "salesforce_cdc": { "max_parallel_snapshot_tables": "uses max_parallel_snapshot_objects; migrate", }, - "tigerbeetle_cdc": { - "checkpoint_cache": "non-relational; §5 applicability under triage", - "checkpoint_limit": "non-relational; §5 applicability under triage", - "snapshot_max_batch_size": "non-relational; §5 applicability under triage", - "max_parallel_snapshot_tables": "non-relational; §5 applicability under triage", - "stream_snapshot": "non-relational; §5 applicability under triage", - }, } // componentSchema is the subset of the docs.ComponentSpec JSON emitted by From 0aa387adfd3d9a05ecf4ed8464e6584bd1267d71 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Wed, 15 Jul 2026 15:35:56 +0300 Subject: [PATCH 08/48] test(cdctest): waive tigerbeetle_cdc conformance fields --- .../plugins/cdctest/cdc_conformance_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/internal/plugins/cdctest/cdc_conformance_test.go b/internal/plugins/cdctest/cdc_conformance_test.go index 31141ddc2a..6a3980caba 100644 --- a/internal/plugins/cdctest/cdc_conformance_test.go +++ b/internal/plugins/cdctest/cdc_conformance_test.go @@ -54,6 +54,13 @@ var canonicalFields = []string{ "stream_snapshot", } +// conditionalConnectors lists CDC inputs that are only registered under specific +// build tags (e.g. cgo). They are exempt from the stale-entry guard when not +// registered, but are still subject to conformance checks when they are. +var conditionalConnectors = map[string]bool{ + "tigerbeetle_cdc": true, // requires cgo +} + // knownNonConformant waives specific (connector → field → reason) checks that // have not yet been migrated. New connectors default to strict. Populated from // the actual registry state; shrink it as connectors converge. @@ -92,6 +99,13 @@ var knownNonConformant = map[string]map[string]string{ "salesforce_cdc": { "max_parallel_snapshot_tables": "uses max_parallel_snapshot_objects; migrate", }, + "tigerbeetle_cdc": { + "checkpoint_cache": "uses progress_cache; migrate to checkpoint_cache", + "checkpoint_limit": "no discrete checkpoint limit; TigerBeetle CDC is pure streaming", + "snapshot_max_batch_size": "no snapshot phase; TigerBeetle CDC is pure streaming with no initial snapshot", + "max_parallel_snapshot_tables": "no snapshot phase; TigerBeetle CDC is pure streaming with no initial snapshot", + "stream_snapshot": "no snapshot phase; TigerBeetle CDC is pure streaming with no initial snapshot", + }, } // componentSchema is the subset of the docs.ComponentSpec JSON emitted by @@ -172,6 +186,10 @@ func TestCDCConformance(t *testing.T) { } for name := range knownNonConformant { if _, ok := registered[name]; !ok { + if conditionalConnectors[name] { + t.Logf("SKIPPED stale check for %q: not registered in this build (conditional build tag)", name) + continue + } t.Errorf("stale knownNonConformant entry %q is not a registered CDC input; remove it", name) } } From ba7c3849c22472df2dbd46d92abce94610903f54 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Wed, 15 Jul 2026 17:17:41 +0300 Subject: [PATCH 09/48] postgres_cdc: fix lint --- internal/impl/postgresql/input_pg_stream.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index deb3edc5bb..11ad70d6c4 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -408,7 +408,7 @@ func validateSchemaPattern(s string) error { return fmt.Errorf("invalid character %q at position %d in schema pattern %q", ch, i, s) } first := rune(s[0]) - if !(first == '_' || first == '*' || (first >= 'a' && first <= 'z') || (first >= 'A' && first <= 'Z')) { + if first != '_' && first != '*' && (first < 'a' || first > 'z') && (first < 'A' || first > 'Z') { return fmt.Errorf("schema pattern %q must start with a letter, underscore, or '*'", s) } return nil From caa0b1ace5d20b919def4ef14191a438f7b67abe Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Wed, 22 Jul 2026 23:42:19 +0300 Subject: [PATCH 10/48] postgres_cdc: skip missing tables per-schema instead of failing whole publication --- .../components/pages/inputs/postgres_cdc.adoc | 2 + internal/impl/postgresql/input_pg_stream.go | 4 +- internal/impl/postgresql/integration_test.go | 75 +++++++++++++++++++ .../pglogicalstream/logical_stream.go | 11 +++ .../pglogicalstream/schema_resolver.go | 37 +++++++++ 5 files changed, 128 insertions(+), 1 deletion(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 069c0f1bfe..70818a8a1d 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -198,6 +198,8 @@ schema: '*' A list of table names to include in the logical replication. Each table should be specified as a separate item. +When `schema` is a glob pattern, this list is resolved against each matched schema independently: a table missing from a given schema is skipped (with a warning logged) rather than failing replication for every matched schema. + *Type*: `array` diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 11ad70d6c4..0ad7e1c887 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -122,7 +122,9 @@ Schema pattern matching runs once at pipeline startup. Schemas created after the Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`, "tenant_*", "*"), ). Field(service.NewStringListField(fieldTables). - Description("A list of table names to include in the logical replication. Each table should be specified as a separate item."). + Description(`A list of table names to include in the logical replication. Each table should be specified as a separate item. + +When `+"`schema`"+` is a glob pattern, this list is resolved against each matched schema independently: a table missing from a given schema is skipped (with a warning logged) rather than failing replication for every matched schema.`). Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`})). Field(service.NewIntField(fieldCheckpointLimit). Description("The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given LSN will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees."). diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 55ebafe4f1..344c20f006 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1738,6 +1738,81 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } +func TestIntegrationMultiSchemaMissingTableDegradesGracefully(t *testing.T) { + integration.CheckSkip(t) + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + // tenant_a is fully provisioned with the "events" table; tenant_b matches + // the schema glob but is missing it (e.g. still being migrated). Before + // this fix, CreatePublication's FOR TABLE clause would reference the + // non-existent tenant_b.events relation and fail publication setup for + // every matched schema, not just the drifted one. + _, err = db.Exec("CREATE SCHEMA tenant_a") + require.NoError(t, err) + _, err = db.Exec("CREATE TABLE tenant_a.events (id SERIAL PRIMARY KEY, name TEXT)") + require.NoError(t, err) + _, err = db.Exec("CREATE SCHEMA tenant_b") + require.NoError(t, err) + + _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('alice')") + require.NoError(t, err) + + type msgMeta struct { + pgSchema string + table string + } + + var ( + mu sync.Mutex + collected []msgMeta + ) + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: missing_table_degrade_slot + stream_snapshot: true + schema: tenant_* + tables: + - events +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + m := msgMeta{} + m.pgSchema, _ = msg.MetaGet("pg_schema") + m.table, _ = msg.MetaGet("table") + collected = append(collected, m) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { _ = stream.Run(t.Context()) }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + // tenant_a should keep streaming even though tenant_b is missing the table. + assert.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(collected) >= 1 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for tenant_a snapshot row; a missing table in tenant_b should not block replication") + + mu.Lock() + defer mu.Unlock() + require.Len(t, collected, 1) + assert.Equal(t, "tenant_a", collected[0].pgSchema) + assert.Equal(t, "events", collected[0].table) +} + func TestIntegrationNoSchemasMatchedError(t *testing.T) { integration.CheckSkip(t) databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 1c0ecf322d..96f4953d46 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -118,10 +118,21 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { tables := make([]TableFQN, 0, len(schemas)*len(normalizedTables)) for _, schema := range schemas { + existingTables, err := resolveExistingTables(ctx, dbConn, schema) + if err != nil { + return nil, fmt.Errorf("resolving tables in schema %q: %w", schema, err) + } for _, table := range normalizedTables { + if _, ok := existingTables[table]; !ok { + config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched pattern %q but does not contain this table)", schema, table, schema, config.DBSchemaPattern) + continue + } tables = append(tables, TableFQN{Schema: schema, Table: table}) } } + if len(tables) == 0 && len(normalizedTables) > 0 { + return nil, fmt.Errorf("none of the configured tables %v were found in any schema matching pattern %q", config.DBTables, config.DBSchemaPattern) + } batchSize := 1000 if config.BatchSize > 0 { batchSize = config.BatchSize diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 8920121e99..117b822da9 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -82,6 +82,43 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) ([ return schemas, nil } +// resolveExistingTables returns the set of quoted table identifiers that +// actually exist in the given (already quoted) schema. +// +// This is used to resolve a schema glob × table list combination per-schema +// rather than assuming every matched schema contains every listed table. A +// schema matching the glob but missing one of the configured tables (e.g. a +// tenant schema that's still being provisioned) would otherwise cause +// CreatePublication's FOR TABLE clause to reference a non-existent relation, +// failing publication setup for every schema, not just the drifted one. +func resolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchema string) (map[string]struct{}, error) { + schema, err := sanitize.UnquotePostgresIdentifier(quotedSchema) + if err != nil { + return nil, fmt.Errorf("unquoting schema identifier %q: %w", quotedSchema, err) + } + + q, err := sanitize.SQLQuery( + "SELECT table_name FROM information_schema.tables WHERE table_schema = $1", + schema, + ) + if err != nil { + return nil, fmt.Errorf("building table resolution query for schema %q: %w", quotedSchema, err) + } + + results, err := conn.Exec(ctx, q).ReadAll() + if err != nil { + return nil, fmt.Errorf("querying tables in schema %q: %w", quotedSchema, err) + } + + existing := map[string]struct{}{} + if len(results) > 0 { + for _, row := range results[0].Rows { + existing[sanitize.QuotePostgresIdentifier(string(row[0]))] = struct{}{} + } + } + return existing, nil +} + // globToLike converts an unquoted glob pattern (using '*' as wildcard) into a // PostgreSQL LIKE pattern that uses '!' as the escape character. // From c161947c20bf8051a5fe14e9020beafb446346d9 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Wed, 22 Jul 2026 23:48:02 +0300 Subject: [PATCH 11/48] postgres_cdc: fix lint --- internal/gateway/authz_endpoint_test.go | 6 +++--- internal/impl/otlp/mock_policy_server_test.go | 6 +++--- internal/impl/postgresql/input_pg_stream.go | 2 +- internal/impl/protobuf/processor_protobuf_test.go | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/gateway/authz_endpoint_test.go b/internal/gateway/authz_endpoint_test.go index b7f240b4f6..899e04b048 100644 --- a/internal/gateway/authz_endpoint_test.go +++ b/internal/gateway/authz_endpoint_test.go @@ -20,7 +20,7 @@ import ( policymaterializerv1 "buf.build/gen/go/redpandadata/common/protocolbuffers/go/redpanda/policymaterializer/v1" "connectrpc.com/connect" "golang.org/x/net/http2" - "golang.org/x/net/http2/h2c" //nolint:staticcheck + "golang.org/x/net/http2/h2c" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -65,8 +65,8 @@ func startPolicyMaterializerServer(t *testing.T, svc policymaterializerv1connect lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") require.NoError(t, err) - srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})} //nolint:staticcheck - go srv.Serve(lis) //nolint:errcheck // test server + srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})} + go srv.Serve(lis) //nolint:errcheck // test server t.Cleanup(func() { srv.Close() }) return "http://" + lis.Addr().String() diff --git a/internal/impl/otlp/mock_policy_server_test.go b/internal/impl/otlp/mock_policy_server_test.go index 43ae225c7c..64d5781551 100644 --- a/internal/impl/otlp/mock_policy_server_test.go +++ b/internal/impl/otlp/mock_policy_server_test.go @@ -18,7 +18,7 @@ import ( policymaterializerv1 "buf.build/gen/go/redpandadata/common/protocolbuffers/go/redpanda/policymaterializer/v1" "connectrpc.com/connect" "golang.org/x/net/http2" - "golang.org/x/net/http2/h2c" //nolint:staticcheck + "golang.org/x/net/http2/h2c" "github.com/stretchr/testify/require" ) @@ -59,8 +59,8 @@ func startMockPolicyEndpoint(t *testing.T, svc policymaterializerv1connect.Polic lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") require.NoError(t, err) - srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})} //nolint:staticcheck - go srv.Serve(lis) //nolint:errcheck + srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})} + go srv.Serve(lis) //nolint:errcheck t.Cleanup(func() { srv.Close() }) return "http://" + lis.Addr().String() diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 0ad7e1c887..1bbac06ed1 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -124,7 +124,7 @@ Schema pattern matching runs once at pipeline startup. Schemas created after the Field(service.NewStringListField(fieldTables). Description(`A list of table names to include in the logical replication. Each table should be specified as a separate item. -When `+"`schema`"+` is a glob pattern, this list is resolved against each matched schema independently: a table missing from a given schema is skipped (with a warning logged) rather than failing replication for every matched schema.`). +When ` + "`schema`" + ` is a glob pattern, this list is resolved against each matched schema independently: a table missing from a given schema is skipped (with a warning logged) rather than failing replication for every matched schema.`). Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`})). Field(service.NewIntField(fieldCheckpointLimit). Description("The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given LSN will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees."). diff --git a/internal/impl/protobuf/processor_protobuf_test.go b/internal/impl/protobuf/processor_protobuf_test.go index d7f6fd01d9..92d8f6bd90 100644 --- a/internal/impl/protobuf/processor_protobuf_test.go +++ b/internal/impl/protobuf/processor_protobuf_test.go @@ -51,7 +51,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/http2" - "golang.org/x/net/http2/h2c" //nolint:staticcheck + "golang.org/x/net/http2/h2c" "google.golang.org/protobuf/types/descriptorpb" "github.com/redpanda-data/benthos/v4/public/service" @@ -463,7 +463,7 @@ func runMockBSRServer(t *testing.T, importPath string) string { fileDescriptorSetServer := &fileDescriptorSetServer{fileDescriptorSet: files} mux.Handle(reflectv1beta1connect.NewFileDescriptorSetServiceHandler(fileDescriptorSetServer)) go func() { - if err := http.Serve(listener, h2c.NewHandler(mux, &http2.Server{})); err != nil && !errors.Is(err, http.ErrServerClosed) { //nolint:staticcheck + if err := http.Serve(listener, h2c.NewHandler(mux, &http2.Server{})); err != nil && !errors.Is(err, http.ErrServerClosed) { require.NoError(t, err) } }() From 0ed96fdf0cd8364c58b62e55963d8cc7cb208fbb Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Thu, 23 Jul 2026 13:03:26 +0300 Subject: [PATCH 12/48] Update internal/impl/postgresql/pglogicalstream/schema_resolver.go Co-authored-by: Joseph Woodward --- internal/impl/postgresql/pglogicalstream/schema_resolver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 117b822da9..8f547bcff7 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -1,4 +1,4 @@ -// Copyright 2024 Redpanda Data, Inc. +// Copyright 2026 Redpanda Data, Inc. // // Licensed as a Redpanda Enterprise file under the Redpanda Community // License (the "License"); you may not use this file except in compliance with From d893e1ac9bc9efc93464b6e776ffea9d240e3884 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Mon, 27 Jul 2026 15:38:08 +0300 Subject: [PATCH 13/48] postgres_cdc: warn when schema pattern matches privilege-hidden schemas information_schema.schemata only lists schemas the connecting role can see, so a schema hidden by missing USAGE was silently dropped from a matched pattern with no signal to the user. resolveSchemas now cross-checks pg_catalog.pg_namespace, which isn't privilege-filtered, and reports those as inaccessible so logical_stream.go can warn instead of skipping silently. --- .../pglogicalstream/logical_stream.go | 5 +- .../pglogicalstream/schema_resolver.go | 49 +++++++++++-- .../schema_resolver_integration_test.go | 71 +++++++++++++++++++ 3 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 96f4953d46..b7cf77d571 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -98,10 +98,13 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { return nil, err } - schemas, err := resolveSchemas(ctx, dbConn, config.DBSchemaPattern) + schemas, inaccessibleSchemas, err := resolveSchemas(ctx, dbConn, config.DBSchemaPattern) if err != nil { return nil, fmt.Errorf("resolving schema pattern %q: %w", config.DBSchemaPattern, err) } + if len(inaccessibleSchemas) > 0 { + config.Logger.Warnf("schema pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaPattern, inaccessibleSchemas) + } if len(schemas) == 0 { return nil, fmt.Errorf("no schemas found matching pattern %q", config.DBSchemaPattern) } diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 8f547bcff7..3859ccface 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -51,10 +51,17 @@ func schemaPatternToLike(pattern string) (string, error) { return globToLike(strings.ToLower(pattern)), nil } -func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) ([]string, error) { +// resolveSchemas returns the schemas matching pattern that the connection's +// role has access to (visibleSchemas), plus any schemas that also match +// pattern but are hidden from information_schema.schemata by privileges +// (inaccessibleSchemas). The latter is surfaced separately so callers can +// warn the user instead of silently dropping schemas they expected to be +// included, e.g. `"tenant_*"` matching a schema the configured role can't +// see yet. +func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (visibleSchemas, inaccessibleSchemas []string, err error) { likePattern, err := schemaPatternToLike(pattern) if err != nil { - return nil, err + return nil, nil, err } q, err := sanitize.SQLQuery( @@ -62,24 +69,54 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) ([ likePattern, ) if err != nil { - return nil, fmt.Errorf("building schema resolution query: %w", err) + return nil, nil, fmt.Errorf("building schema resolution query: %w", err) } results, err := conn.Exec(ctx, q).ReadAll() if err != nil { - return nil, fmt.Errorf("querying schemas matching %q: %w", pattern, err) + return nil, nil, fmt.Errorf("querying schemas matching %q: %w", pattern, err) } + visible := map[string]struct{}{} var schemas []string if len(results) > 0 { for _, row := range results[0].Rows { + name := string(row[0]) + visible[name] = struct{}{} // QuotePostgresIdentifier preserves the exact stored name (including // case for case-sensitive schemas), unlike NormalizePostgresIdentifier // which would incorrectly fold to lower-case. - schemas = append(schemas, sanitize.QuotePostgresIdentifier(string(row[0]))) + schemas = append(schemas, sanitize.QuotePostgresIdentifier(name)) } } - return schemas, nil + + // pg_namespace is not privilege-filtered, so any pattern match here that's + // missing from information_schema.schemata means the role lacks USAGE (or + // similar) on that schema rather than the schema simply not existing. + nsQ, err := sanitize.SQLQuery( + "SELECT nspname FROM pg_catalog.pg_namespace WHERE nspname LIKE $1 ESCAPE '!' AND nspname NOT LIKE 'pg!_%' ESCAPE '!' AND nspname != 'information_schema'", + likePattern, + ) + if err != nil { + return nil, nil, fmt.Errorf("building pg_namespace resolution query: %w", err) + } + + nsResults, err := conn.Exec(ctx, nsQ).ReadAll() + if err != nil { + return nil, nil, fmt.Errorf("querying pg_namespace for schemas matching %q: %w", pattern, err) + } + + var hidden []string + if len(nsResults) > 0 { + for _, row := range nsResults[0].Rows { + name := string(row[0]) + if _, ok := visible[name]; !ok { + hidden = append(hidden, sanitize.QuotePostgresIdentifier(name)) + } + } + } + + return schemas, hidden, nil } // resolveExistingTables returns the set of quoted table identifiers that diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go b/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go new file mode 100644 index 0000000000..6af1d4ad7e --- /dev/null +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go @@ -0,0 +1,71 @@ +// Copyright 2024 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md + +package pglogicalstream + +import ( + "context" + "database/sql" + "testing" + "time" + + _ "github.com/lib/pq" // registers "postgres" driver for sql.Open in tests + + "github.com/jackc/pgx/v5/pgconn" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service/integration" +) + +// TestIntegrationResolveSchemasReportsInaccessibleSchemas verifies that a +// schema pattern matching a schema the connecting role lacks USAGE on is +// reported via inaccessibleSchemas rather than silently dropped, since +// information_schema.schemata alone would make it indistinguishable from a +// schema that simply doesn't exist. +func TestIntegrationResolveSchemasReportsInaccessibleSchemas(t *testing.T) { + integration.CheckSkip(t) + + _, adminURL := createDockerInstance(t) + + adminDB, err := sql.Open("postgres", adminURL) + require.NoError(t, err) + defer adminDB.Close() + + _, err = adminDB.Exec("CREATE SCHEMA visible_schema") + require.NoError(t, err) + _, err = adminDB.Exec("CREATE SCHEMA hidden_schema") + require.NoError(t, err) + + _, err = adminDB.Exec("CREATE ROLE restricted_role LOGIN PASSWORD 'restricted_pw'") + require.NoError(t, err) + _, err = adminDB.Exec("GRANT CONNECT ON DATABASE dbname TO restricted_role") + require.NoError(t, err) + _, err = adminDB.Exec("GRANT USAGE ON SCHEMA visible_schema TO restricted_role") + require.NoError(t, err) + // Deliberately no GRANT on hidden_schema. + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + restrictedConfig, err := pgconn.ParseConfig(adminURL) + require.NoError(t, err) + restrictedConfig.User = "restricted_role" + restrictedConfig.Password = "restricted_pw" + delete(restrictedConfig.RuntimeParams, "replication") + + restrictedConn, err := pgconn.ConnectConfig(ctx, restrictedConfig) + require.NoError(t, err) + defer closeConn(t, restrictedConn) + + visible, inaccessible, err := resolveSchemas(ctx, restrictedConn, "*_schema") + require.NoError(t, err) + + assert.Equal(t, []string{`"visible_schema"`}, visible) + assert.Equal(t, []string{`"hidden_schema"`}, inaccessible) +} From 0e662de9d78a8903fcb2a77d0a709127442ae83f Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 4 Aug 2026 12:15:05 +0100 Subject: [PATCH 14/48] postgres_cdc: Address minor issues --- docs/modules/components/pages/inputs/postgres_cdc.adoc | 1 + internal/gateway/authz_endpoint_test.go | 6 +++--- internal/impl/otlp/mock_policy_server_test.go | 6 +++--- internal/impl/protobuf/processor_protobuf_test.go | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 37adc7fe1b..849adbd8ae 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -199,6 +199,7 @@ schema: '*' A list of table names to include in the logical replication. Each table should be specified as a separate item. When `schema` is a glob pattern, this list is resolved against each matched schema independently: a table missing from a given schema is skipped (with a warning logged) rather than failing replication for every matched schema. + If left empty, the underlying PostgreSQL publication is created `FOR ALL TABLES`, which replicates every table in every schema of the database, ignoring `schema`. This also disables `stream_snapshot`, since the initial snapshot is only planned for tables listed here. diff --git a/internal/gateway/authz_endpoint_test.go b/internal/gateway/authz_endpoint_test.go index 899e04b048..b7f240b4f6 100644 --- a/internal/gateway/authz_endpoint_test.go +++ b/internal/gateway/authz_endpoint_test.go @@ -20,7 +20,7 @@ import ( policymaterializerv1 "buf.build/gen/go/redpandadata/common/protocolbuffers/go/redpanda/policymaterializer/v1" "connectrpc.com/connect" "golang.org/x/net/http2" - "golang.org/x/net/http2/h2c" + "golang.org/x/net/http2/h2c" //nolint:staticcheck "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -65,8 +65,8 @@ func startPolicyMaterializerServer(t *testing.T, svc policymaterializerv1connect lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") require.NoError(t, err) - srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})} - go srv.Serve(lis) //nolint:errcheck // test server + srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})} //nolint:staticcheck + go srv.Serve(lis) //nolint:errcheck // test server t.Cleanup(func() { srv.Close() }) return "http://" + lis.Addr().String() diff --git a/internal/impl/otlp/mock_policy_server_test.go b/internal/impl/otlp/mock_policy_server_test.go index 64d5781551..43ae225c7c 100644 --- a/internal/impl/otlp/mock_policy_server_test.go +++ b/internal/impl/otlp/mock_policy_server_test.go @@ -18,7 +18,7 @@ import ( policymaterializerv1 "buf.build/gen/go/redpandadata/common/protocolbuffers/go/redpanda/policymaterializer/v1" "connectrpc.com/connect" "golang.org/x/net/http2" - "golang.org/x/net/http2/h2c" + "golang.org/x/net/http2/h2c" //nolint:staticcheck "github.com/stretchr/testify/require" ) @@ -59,8 +59,8 @@ func startMockPolicyEndpoint(t *testing.T, svc policymaterializerv1connect.Polic lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") require.NoError(t, err) - srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})} - go srv.Serve(lis) //nolint:errcheck + srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})} //nolint:staticcheck + go srv.Serve(lis) //nolint:errcheck t.Cleanup(func() { srv.Close() }) return "http://" + lis.Addr().String() diff --git a/internal/impl/protobuf/processor_protobuf_test.go b/internal/impl/protobuf/processor_protobuf_test.go index 92d8f6bd90..d7f6fd01d9 100644 --- a/internal/impl/protobuf/processor_protobuf_test.go +++ b/internal/impl/protobuf/processor_protobuf_test.go @@ -51,7 +51,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/http2" - "golang.org/x/net/http2/h2c" + "golang.org/x/net/http2/h2c" //nolint:staticcheck "google.golang.org/protobuf/types/descriptorpb" "github.com/redpanda-data/benthos/v4/public/service" @@ -463,7 +463,7 @@ func runMockBSRServer(t *testing.T, importPath string) string { fileDescriptorSetServer := &fileDescriptorSetServer{fileDescriptorSet: files} mux.Handle(reflectv1beta1connect.NewFileDescriptorSetServiceHandler(fileDescriptorSetServer)) go func() { - if err := http.Serve(listener, h2c.NewHandler(mux, &http2.Server{})); err != nil && !errors.Is(err, http.ErrServerClosed) { + if err := http.Serve(listener, h2c.NewHandler(mux, &http2.Server{})); err != nil && !errors.Is(err, http.ErrServerClosed) { //nolint:staticcheck require.NoError(t, err) } }() From e7dccc60b99d32aaf670db8cfe03a7dc82d7c45c Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 4 Aug 2026 12:53:19 +0100 Subject: [PATCH 15/48] postgres_cdc: fix broken test --- internal/impl/postgresql/integration_test.go | 35 ++++++++++---------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index f675014071..f61148c4a4 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1831,35 +1831,36 @@ postgres_cdc: assert.Equal(t, "events", collected[0].table) } -func TestIntegrationNoSchemasMatchedError(t *testing.T) { +func TestIntegrationNoSchemasMatchedReturnsError(t *testing.T) { integration.CheckSkip(t) databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) tmpl := fmt.Sprintf(` -postgres_cdc: - dsn: %s - slot_name: no_schema_match_slot - schema: nonexistent_schema_zzz_* - tables: - - events +dsn: %s +slot_name: no_schema_match_slot +schema: nonexistent_schema_zzz_* +tables: + - events `, databaseURL) - sb := service.NewStreamBuilder() - require.NoError(t, sb.SetLoggerYAML(`level: ERROR`)) - require.NoError(t, sb.AddInputYAML(tmpl)) - require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, _ service.MessageBatch) error { - return nil - })) + conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) + require.NoError(t, err) - stream, err := sb.Build() + mgr := service.MockResources() + license.InjectTestService(mgr) + + input, err := newPgStreamInput(conf, mgr) require.NoError(t, err) - license.InjectTestService(stream.Resources()) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - err = stream.Run(ctx) + // Bypass the benthos AsyncReader's infinite connect-retry loop by calling + // Connect directly: a schema-pattern-not-found error is permanent, but + // stream.Run has no path to surface it (it only returns once ctx is done), + // so going through StreamBuilder/Run here would just time out instead. + err = input.Connect(ctx) require.Error(t, err) assert.Contains(t, err.Error(), "no schemas found matching pattern") } From 80e2204e86a555b365b7c5980ec4cf3ab3fdf802 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 4 Aug 2026 13:09:51 +0100 Subject: [PATCH 16/48] postgres_cdc: replace pg_schema with database_schema --- internal/impl/postgresql/input_pg_stream.go | 4 +- internal/impl/postgresql/integration_test.go | 70 +++++++++---------- .../impl/postgresql/tests/current/setup.sql | 2 +- .../postgresql/tests/current/test_config.yaml | 4 +- 4 files changed, 40 insertions(+), 40 deletions(-) diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 13c0a41fd6..5cecc17457 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -85,7 +85,7 @@ Additionally, if ` + "`" + fieldStreamSnapshot + "`" + ` is set to true, then th This input adds the following metadata fields to each message: - table: Name of the table that the message originated from -- pg_schema: The PostgreSQL schema name that the table belongs to (e.g. "public", "tenant_foo"). Useful for per-schema routing when using schema patterns. +- database_schema: The database schema for the table where the message originates from (e.g. "public", "tenant_foo"). Useful for per-schema routing when using schema patterns. - operation: Type of operation that generated the message: "read", "insert", "update", or "delete". "read" is from messages that are read in the initial snapshot phase. This will also be "begin" and "commit" if ` + "`" + fieldIncludeTxnMarkers + "`" + ` is enabled - lsn: the log sequence number in postgres - schema: The table schema in benthos common schema format, compatible with processors like parquet_encode @@ -586,7 +586,7 @@ func (p *pgStreamInput) processStream(pgStream *pglogicalstream.Stream, batcher } batchMsg := service.NewMessage(mb) batchMsg.MetaSet("table", msg.Table) - batchMsg.MetaSet("pg_schema", msg.Schema) + batchMsg.MetaSet("database_schema", msg.Schema) batchMsg.MetaSet("operation", string(msg.Operation)) if msg.LSN != nil { batchMsg.MetaSet("lsn", *msg.LSN) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index f61148c4a4..e183f0f517 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1249,44 +1249,44 @@ postgres_cdc: outBatches, []any{ map[string]any{ - "operation": "read", - "table": "FlightsCompositePK", - "pg_schema": "public", + "operation": "read", + "table": "FlightsCompositePK", + "database_schema": "public", }, map[string]any{ - "operation": "read", - "table": "flights", - "pg_schema": "public", + "operation": "read", + "table": "flights", + "database_schema": "public", }, map[string]any{ - "operation": "insert", - "table": "FlightsCompositePK", - "lsn": "XXX/XXX", - "commit_ts_ms": "SET", - "pg_schema": "public", + "operation": "insert", + "table": "FlightsCompositePK", + "lsn": "XXX/XXX", + "commit_ts_ms": "SET", + "database_schema": "public", }, map[string]any{ - "operation": "insert", - "table": "flights", - "lsn": "XXX/XXX", - "commit_ts_ms": "SET", - "pg_schema": "public", + "operation": "insert", + "table": "flights", + "lsn": "XXX/XXX", + "commit_ts_ms": "SET", + "database_schema": "public", }, map[string]any{ - "operation": "update", - "table": "flights", - "lsn": "XXX/XXX", - "commit_ts_ms": "SET", - "before": "SET", - "pg_schema": "public", + "operation": "update", + "table": "flights", + "lsn": "XXX/XXX", + "commit_ts_ms": "SET", + "before": "SET", + "database_schema": "public", }, map[string]any{ - "operation": "delete", - "table": "flights", - "lsn": "XXX/XXX", - "commit_ts_ms": "SET", - "before": "SET", - "pg_schema": "public", + "operation": "delete", + "table": "flights", + "lsn": "XXX/XXX", + "commit_ts_ms": "SET", + "before": "SET", + "database_schema": "public", }, }, ) @@ -1656,7 +1656,7 @@ func TestIntegrationMultiSchemaSnapshotAndCDC(t *testing.T) { require.NoError(t, err) type msgMeta struct { - pgSchema string + dbSchema string table string operation string lsn string @@ -1685,7 +1685,7 @@ postgres_cdc: defer mu.Unlock() for _, msg := range batch { m := msgMeta{} - m.pgSchema, _ = msg.MetaGet("pg_schema") + m.dbSchema, _ = msg.MetaGet("database_schema") m.table, _ = msg.MetaGet("table") m.operation, _ = msg.MetaGet("operation") m.lsn, _ = msg.MetaGet("lsn") @@ -1738,7 +1738,7 @@ postgres_cdc: for _, m := range snapshots { assert.Equal(t, "events", m.table, "snapshot: table should be bare name without schema prefix") assert.Empty(t, m.lsn, "snapshot rows have no LSN") - snapshotSchemas[m.pgSchema]++ + snapshotSchemas[m.dbSchema]++ } assert.Equal(t, 2, snapshotSchemas["tenant_a"], "expected 2 snapshot rows from tenant_a") assert.Equal(t, 1, snapshotSchemas["tenant_b"], "expected 1 snapshot row from tenant_b") @@ -1750,7 +1750,7 @@ postgres_cdc: assert.Equal(t, "insert", m.operation) assert.Equal(t, "events", m.table) assert.NotEmpty(t, m.lsn, "CDC rows must have an LSN") - cdcSchemas[m.pgSchema]++ + cdcSchemas[m.dbSchema]++ } assert.Equal(t, 1, cdcSchemas["tenant_a"], "expected 1 CDC row from tenant_a") assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") @@ -1777,7 +1777,7 @@ func TestIntegrationMultiSchemaMissingTableDegradesGracefully(t *testing.T) { require.NoError(t, err) type msgMeta struct { - pgSchema string + dbSchema string table string } @@ -1804,7 +1804,7 @@ postgres_cdc: defer mu.Unlock() for _, msg := range batch { m := msgMeta{} - m.pgSchema, _ = msg.MetaGet("pg_schema") + m.dbSchema, _ = msg.MetaGet("database_schema") m.table, _ = msg.MetaGet("table") collected = append(collected, m) } @@ -1827,7 +1827,7 @@ postgres_cdc: mu.Lock() defer mu.Unlock() require.Len(t, collected, 1) - assert.Equal(t, "tenant_a", collected[0].pgSchema) + assert.Equal(t, "tenant_a", collected[0].dbSchema) assert.Equal(t, "events", collected[0].table) } diff --git a/internal/impl/postgresql/tests/current/setup.sql b/internal/impl/postgresql/tests/current/setup.sql index ee40a376cc..512f8373fa 100644 --- a/internal/impl/postgresql/tests/current/setup.sql +++ b/internal/impl/postgresql/tests/current/setup.sql @@ -1,5 +1,5 @@ -- Multi-schema CDC test setup --- Tests: schema glob (tenant_*), pg_schema metadata, commit_ts_ms, before (update/delete) +-- Tests: schema glob (tenant_*), database_schema metadata, commit_ts_ms, before (update/delete) -- ── Tenant schemas ──────────────────────────────────────────────────────────── diff --git a/internal/impl/postgresql/tests/current/test_config.yaml b/internal/impl/postgresql/tests/current/test_config.yaml index 4a994e30c2..9e40b5a1c1 100644 --- a/internal/impl/postgresql/tests/current/test_config.yaml +++ b/internal/impl/postgresql/tests/current/test_config.yaml @@ -15,14 +15,14 @@ pipeline: - mapping: | let op = @operation let tbl = @table - let schema = @pg_schema + let schema = @database_schema let lsn = @lsn let ts_ms = @commit_ts_ms let before = @before root = { "operation": $op, - "pg_schema": $schema, + "database_schema": $schema, "table": $tbl, "payload": this, "lsn": if $lsn != null { $lsn } else { null }, From fc960ceae39e2fbef4457a71fef2311f4dac28d6 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 4 Aug 2026 13:20:58 +0100 Subject: [PATCH 17/48] postgres_cdc: move schema validation to unit test closer to use --- .../impl/postgresql/input_pg_stream_test.go | 72 +++++++++ .../tests/schema_validation/validate_test.go | 147 ------------------ 2 files changed, 72 insertions(+), 147 deletions(-) create mode 100644 internal/impl/postgresql/input_pg_stream_test.go delete mode 100644 internal/impl/postgresql/tests/schema_validation/validate_test.go diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go new file mode 100644 index 0000000000..19d9723bc2 --- /dev/null +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -0,0 +1,72 @@ +// Copyright 2024 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md + +package pgstream + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + + "github.com/redpanda-data/connect/v4/internal/license" +) + +// TestSchemaPatternValidation verifies that the schema field is validated +// during config parsing, before any network I/O is attempted. Success is +// asserted via newPgStreamInput returning no error - the constructor doesn't +// dial the database, so a valid pattern implies validation passed. +func TestSchemaPatternValidation(t *testing.T) { + tests := []struct { + pattern string + errContains string + }{ + {"public", ""}, + {"tenant_*", ""}, + {"*", ""}, + {`"MySchema"`, ""}, + // Regression test: len("") == 2 used to pass the old `len(s) < 2` guard. + // Fixed to `len(s) < 3`. + {`""`, "invalid quoted schema identifier"}, + {"1abc", "must start with a letter"}, + {`"unclosed`, "invalid quoted schema identifier"}, + {"schema-name", "invalid character"}, + {"", "schema cannot be empty"}, + {`"quoted*"`, "wildcard"}, + } + for _, tt := range tests { + t.Run(tt.pattern, func(t *testing.T) { + // Single-quoted so the pattern (which may itself contain double + // quotes, e.g. `"MySchema"`) reaches validateSchemaPattern verbatim. + yaml := fmt.Sprintf(` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +schema: '%s' +slot_name: test_slot +tables: + - events +`, tt.pattern) + + conf, err := newPostgresCDCConfig().ParseYAML(yaml, nil) + require.NoError(t, err) + + mgr := service.MockResources() + license.InjectTestService(mgr) + + _, err = newPgStreamInput(conf, mgr) + if tt.errContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errContains) + return + } + require.NoError(t, err) + }) + } +} diff --git a/internal/impl/postgresql/tests/schema_validation/validate_test.go b/internal/impl/postgresql/tests/schema_validation/validate_test.go deleted file mode 100644 index e45e4bf2f2..0000000000 --- a/internal/impl/postgresql/tests/schema_validation/validate_test.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2024 Redpanda Data, Inc. -// -// Licensed as a Redpanda Enterprise file under the Redpanda Community -// License (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md - -// Package schema_validation tests postgres_cdc schema pattern validation -// through the public service config API. No database connection is required: -// invalid schemas are rejected during stream construction (before any network -// I/O), so stream.Run returns synchronously for the invalid-schema cases. -package schema_validation_test - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - _ "github.com/redpanda-data/benthos/v4/public/components/pure" // registers none tracer and other built-ins - "github.com/redpanda-data/benthos/v4/public/service" - - _ "github.com/redpanda-data/connect/v4/internal/impl/postgresql" // registers postgres_cdc - "github.com/redpanda-data/connect/v4/internal/license" -) - -// postgresStream builds a postgres_cdc stream with the given YAML schema value -// (the raw fragment that appears after "schema: " in YAML) and injects a test -// enterprise license. Returns the stream ready to run. -func postgresStream(t *testing.T, schemaYAML string) *service.Stream { - t.Helper() - yaml := fmt.Sprintf(` -postgres_cdc: - dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable - schema: %s - slot_name: test_slot - tables: - - events -`, schemaYAML) - - sb := service.NewStreamBuilder() - require.NoError(t, sb.SetLoggerYAML(`level: ERROR`)) - require.NoError(t, sb.AddInputYAML(yaml)) - require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, _ service.MessageBatch) error { - return nil - })) - - stream, err := sb.Build() - require.NoError(t, err) - license.InjectTestService(stream.Resources()) - return stream -} - -// TestInvalidSchemaPatterns verifies that invalid schema values are rejected -// during stream construction — before any database connection is attempted. -// stream.Run returns synchronously (no goroutine needed) when the constructor -// fails. -func TestInvalidSchemaPatterns(t *testing.T) { - tests := []struct { - name string - schemaYAML string - wantErrMsg string - }{ - { - // Regression test: len("") == 2 used to pass the old `len(s) < 2` - // guard. Fixed to `len(s) < 3`. - name: "empty quoted identifier", - schemaYAML: `'""'`, - wantErrMsg: "invalid schema", - }, - { - name: "digit-first unquoted pattern", - schemaYAML: `"1abc"`, - wantErrMsg: "invalid schema", - }, - { - name: "unterminated quoted identifier", - schemaYAML: `'"unclosed'`, - wantErrMsg: "invalid schema", - }, - { - name: "hyphen in unquoted pattern", - schemaYAML: `schema-name`, - wantErrMsg: "invalid schema", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - stream := postgresStream(t, tt.schemaYAML) - - // Run returns synchronously when the input constructor fails — - // no timeout context needed, but use one as a safety net. - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - err := stream.Run(ctx) - require.Error(t, err, "expected stream construction to fail") - assert.Contains(t, err.Error(), tt.wantErrMsg, - "error should indicate schema validation failure") - }) - } -} - -// TestValidSchemaPatterns verifies that valid schema values pass construction -// and are only rejected later (at DB-connect time). We run the stream briefly -// and confirm no "invalid schema" error surfaces. -func TestValidSchemaPatterns(t *testing.T) { - tests := []struct { - name string - schemaYAML string - }{ - {name: "exact unquoted schema", schemaYAML: `public`}, - {name: "glob pattern", schemaYAML: `tenant_*`}, - {name: "wildcard", schemaYAML: `"*"`}, - {name: "quoted exact identifier", schemaYAML: `'"MySchema"'`}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - stream := postgresStream(t, tt.schemaYAML) - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - errCh := make(chan error, 1) - go func() { errCh <- stream.Run(ctx) }() - - select { - case err := <-errCh: - // Stream stopped before timeout — must not be a schema error. - if err != nil { - assert.NotContains(t, err.Error(), "invalid schema", - "valid schema %q should not trigger schema validation error", tt.schemaYAML) - } - case <-ctx.Done(): - // Stream is still running after timeout — constructor succeeded, - // stream is attempting DB connection. This is the expected path. - _ = stream.StopWithin(2 * time.Second) - } - }) - } -} From 3411584a7c1cbe7a1423ed9ef2566431ec63e1fc Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 4 Aug 2026 13:26:13 +0100 Subject: [PATCH 18/48] postgres_cdc: clean up redundant comment --- .../pglogicalstream/schema_resolver.go | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 3859ccface..4737f7c4cb 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -18,22 +18,6 @@ import ( "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/sanitize" ) -// resolveSchemas expands a schema name or glob pattern into the set of -// quoted PostgreSQL identifiers that exist in the database. -// -// For unquoted patterns (e.g. "tenant_*") the pattern is matched -// case-insensitively against information_schema.schemata using LIKE, because -// PostgreSQL folds unquoted identifiers to lower-case at creation time. -// -// For quoted identifiers (e.g. `"MySchema"`) an exact case-sensitive lookup -// is performed. -// -// System schemas (pg_* and information_schema) are always excluded so that -// wildcard patterns like "*" do not attempt to replicate catalog tables. -// -// Returns an error if the query fails. Returns (nil, nil) if no schemas match. -// The caller is responsible for treating an empty result as an error. - // schemaPatternToLike converts a schema name or glob pattern into the LIKE // pattern used by resolveSchemas. Extracted for unit testing. // @@ -58,6 +42,17 @@ func schemaPatternToLike(pattern string) (string, error) { // warn the user instead of silently dropping schemas they expected to be // included, e.g. `"tenant_*"` matching a schema the configured role can't // see yet. +// +// For unquoted patterns (e.g. "tenant_*") the pattern is matched +// case-insensitively via LIKE, because PostgreSQL folds unquoted identifiers +// to lower-case at creation time. For quoted identifiers (e.g. `"MySchema"`) +// an exact case-sensitive lookup is performed. System schemas (pg_* and +// information_schema) are always excluded so that wildcard patterns like "*" +// do not attempt to replicate catalog tables. +// +// Returned schema names are quoted PostgreSQL identifiers. Returns an error +// if either query fails; returns a nil visibleSchemas slice (with a nil err) +// if no schemas match — callers should treat that as an error condition. func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (visibleSchemas, inaccessibleSchemas []string, err error) { likePattern, err := schemaPatternToLike(pattern) if err != nil { From bf289596b5a13ba1b210eb5dfdf8ed2261f2c10e Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 4 Aug 2026 13:34:44 +0100 Subject: [PATCH 19/48] postgres_cdc: normalie test structure --- internal/impl/postgresql/integration_test.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index e183f0f517..26217596a5 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1697,7 +1697,11 @@ postgres_cdc: stream, err := sb.Build() require.NoError(t, err) license.InjectTestService(stream.Resources()) - go func() { _ = stream.Run(t.Context()) }() + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) // Wait for all 3 snapshot rows. @@ -1814,7 +1818,11 @@ postgres_cdc: stream, err := sb.Build() require.NoError(t, err) license.InjectTestService(stream.Resources()) - go func() { _ = stream.Run(t.Context()) }() + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) // tenant_a should keep streaming even though tenant_b is missing the table. From 58d3be8886774b200bea0a7e97b6730cf2a0d97d Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 4 Aug 2026 13:46:46 +0100 Subject: [PATCH 20/48] postgres_cdc: t.Context() --- internal/impl/postgresql/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 26217596a5..e68882e7bc 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1861,7 +1861,7 @@ tables: input, err := newPgStreamInput(conf, mgr) require.NoError(t, err) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() // Bypass the benthos AsyncReader's infinite connect-retry loop by calling From 6803eca6c66378366969e56d025ddb0b02df2951 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Thu, 6 Aug 2026 12:24:36 +0100 Subject: [PATCH 21/48] postgres_cdc: revert to existing behaviour If no schema is found and tables is empty then we want to ensure we create publication for all tables. --- internal/impl/postgresql/integration_test.go | 67 +++++++++++++++++++ .../pglogicalstream/logical_stream.go | 4 +- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index e68882e7bc..9bbd8a8882 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1872,3 +1872,70 @@ tables: require.Error(t, err) assert.Contains(t, err.Error(), "no schemas found matching pattern") } + +// TestIntegrationForAllTablesIgnoresNonMatchingSchemaPattern guards the +// documented behaviour of FOR ALL TABLES mode: when `tables` is left empty, +// `schema` has no effect and must not block startup even if it matches no +// schema in the database. +func TestIntegrationForAllTablesIgnoresNonMatchingSchemaPattern(t *testing.T) { + integration.CheckSkip(t) + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: for_all_tables_ignores_schema_slot + schema: nonexistent_schema_zzz_* +`, databaseURL) + + // FOR ALL TABLES mode disables the initial snapshot, so replication only + // sees rows written after the slot/publication exist. Insert continuously + // (rather than once upfront) so a row lands after startup completes. + writer := asyncroutine.NewPeriodic(100*time.Millisecond, func() { + _, err := db.Exec("INSERT INTO flights (name, created_at) VALUES ('alice', now());") + require.NoError(t, err) + }) + writer.Start() + t.Cleanup(writer.Stop) + + var ( + mu sync.Mutex + collected []string + ) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + table, _ := msg.MetaGet("table") + collected = append(collected, table) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + // Non-matching schema pattern must not block FOR ALL TABLES replication: + // the "flights" insert above should still arrive. + assert.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(collected) >= 1 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for FOR ALL TABLES replication; a non-matching schema pattern should be ignored when tables is empty") + + mu.Lock() + defer mu.Unlock() + assert.Contains(t, collected, "flights") +} diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index b7cf77d571..f2d7bd17f8 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -102,10 +102,10 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { if err != nil { return nil, fmt.Errorf("resolving schema pattern %q: %w", config.DBSchemaPattern, err) } - if len(inaccessibleSchemas) > 0 { + if len(inaccessibleSchemas) > 0 && len(config.DBTables) > 0 { config.Logger.Warnf("schema pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaPattern, inaccessibleSchemas) } - if len(schemas) == 0 { + if len(schemas) == 0 && len(config.DBTables) > 0 { return nil, fmt.Errorf("no schemas found matching pattern %q", config.DBSchemaPattern) } config.Logger.Infof("Schema pattern %q resolved to %d schema(s): %v", config.DBSchemaPattern, len(schemas), schemas) From 045cde2e534650affb661c1a6fd1c7acbe04e7ce Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Thu, 6 Aug 2026 13:43:29 +0100 Subject: [PATCH 22/48] postgres_cdc: improve on missing table coverage --- internal/impl/postgresql/integration_test.go | 234 ++++++++++++++++++ .../pglogicalstream/logical_stream.go | 16 +- 2 files changed, 248 insertions(+), 2 deletions(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 9bbd8a8882..35adb4cc28 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1939,3 +1939,237 @@ postgres_cdc: defer mu.Unlock() assert.Contains(t, collected, "flights") } + +func TestIntegrationSchemaAndTableMatchingTest(t *testing.T) { + integration.CheckSkip(t) + + t.Run("exact schema match with missing table fails", func(t *testing.T) { + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + _, err = db.Exec("CREATE TABLE IF NOT EXISTS orders (id SERIAL PRIMARY KEY, name TEXT);") + require.NoError(t, err) + + tmpl := fmt.Sprintf(` +dsn: %s +slot_name: exact_schema_missing_table_slot +schema: public +tables: + - orders + - ordres +`, databaseURL) + + conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) + require.NoError(t, err) + + mgr := service.MockResources() + license.InjectTestService(mgr) + + input, err := newPgStreamInput(conf, mgr) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + // Bypass the benthos AsyncReader's infinite connect-retry loop, same as + // TestIntegrationNoSchemasMatchedReturnsError. + err = input.Connect(ctx) + require.Error(t, err, "typo'd table %q should fail startup loudly instead of silently streaming only %q", "ordres", "orders") + assert.Contains(t, err.Error(), "ordres") + }) + + t.Run("glob schema matching multiple schemas with one or more missing tables fails", func(t *testing.T) { + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + _, err = db.Exec(` + CREATE SCHEMA tenant_a; + CREATE SCHEMA tenant_b; + + CREATE TABLE tenant_a.orders (id SERIAL PRIMARY KEY, name TEXT); + CREATE TABLE tenant_b.orders (id SERIAL PRIMARY KEY, name TEXT); + `) + require.NoError(t, err) + + tmpl := fmt.Sprintf(` +dsn: %s +slot_name: glob_schema_total_miss_slot +schema: tenant_* +tables: + - orders + - ordres +`, databaseURL) + + conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) + require.NoError(t, err) + + mgr := service.MockResources() + license.InjectTestService(mgr) + + input, err := newPgStreamInput(conf, mgr) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + err = input.Connect(ctx) + require.Error(t, err, "ordres exists in neither tenant_a nor tenant_b, so it should fail startup instead of silently streaming only tenant_a/b.orders") + assert.Contains(t, err.Error(), "ordres") + }) + + t.Run("glob schema matching multiple schemas with matching tables passes", func(t *testing.T) { + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + _, err = db.Exec(` + CREATE SCHEMA tenant_a; + CREATE SCHEMA tenant_b; + + CREATE TABLE tenant_a.orders (id SERIAL PRIMARY KEY, name TEXT); + CREATE TABLE tenant_b.ordres (id SERIAL PRIMARY KEY, name TEXT); + + INSERT INTO tenant_a.orders (name) VALUES ('alice'); + INSERT INTO tenant_b.ordres (name) VALUES ('bob'); + `) + require.NoError(t, err) + + type msgMeta struct { + dbSchema string + table string + } + + var ( + mu sync.Mutex + collected []msgMeta + ) + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: glob_schema_partial_match_slot + stream_snapshot: true + schema: tenant_* + tables: + - orders + - ordres +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + m := msgMeta{} + m.dbSchema, _ = msg.MetaGet("database_schema") + m.table, _ = msg.MetaGet("table") + collected = append(collected, m) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + assert.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(collected) >= 2 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for both tenant_a.orders and tenant_b.ordres snapshot rows") + + mu.Lock() + defer mu.Unlock() + require.Len(t, collected, 2) + assert.Contains(t, collected, msgMeta{dbSchema: "tenant_a", table: "orders"}) + assert.Contains(t, collected, msgMeta{dbSchema: "tenant_b", table: "ordres"}) + }) + + t.Run("glob schema matching multiple schemas with all tables present in all schemas passes", func(t *testing.T) { + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + _, err = db.Exec(` + CREATE SCHEMA tenant_a; + CREATE SCHEMA tenant_b; + + CREATE TABLE tenant_a.orders (id SERIAL PRIMARY KEY, name TEXT); + CREATE TABLE tenant_a.ordres (id SERIAL PRIMARY KEY, name TEXT); + CREATE TABLE tenant_b.orders (id SERIAL PRIMARY KEY, name TEXT); + CREATE TABLE tenant_b.ordres (id SERIAL PRIMARY KEY, name TEXT); + + INSERT INTO tenant_a.orders (name) VALUES ('alice'); + INSERT INTO tenant_a.ordres (name) VALUES ('bob'); + INSERT INTO tenant_b.orders (name) VALUES ('carol'); + INSERT INTO tenant_b.ordres (name) VALUES ('dave'); + `) + require.NoError(t, err) + + type msgMeta struct { + dbSchema string + table string + } + + var ( + mu sync.Mutex + collected []msgMeta + ) + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: glob_schema_full_match_slot + stream_snapshot: true + schema: tenant_* + tables: + - orders + - ordres +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + m := msgMeta{} + m.dbSchema, _ = msg.MetaGet("database_schema") + m.table, _ = msg.MetaGet("table") + collected = append(collected, m) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + assert.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(collected) >= 4 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for all four tenant_{a,b}.{orders,ordres} snapshot rows") + + mu.Lock() + defer mu.Unlock() + require.Len(t, collected, 4) + assert.Contains(t, collected, msgMeta{dbSchema: "tenant_a", table: "orders"}) + assert.Contains(t, collected, msgMeta{dbSchema: "tenant_a", table: "ordres"}) + assert.Contains(t, collected, msgMeta{dbSchema: "tenant_b", table: "orders"}) + assert.Contains(t, collected, msgMeta{dbSchema: "tenant_b", table: "ordres"}) + }) +} diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index f2d7bd17f8..5747803116 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -120,6 +120,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } tables := make([]TableFQN, 0, len(schemas)*len(normalizedTables)) + foundTables := make(map[string]bool, len(normalizedTables)) for _, schema := range schemas { existingTables, err := resolveExistingTables(ctx, dbConn, schema) if err != nil { @@ -131,10 +132,21 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { continue } tables = append(tables, TableFQN{Schema: schema, Table: table}) + foundTables[table] = true } } - if len(tables) == 0 && len(normalizedTables) > 0 { - return nil, fmt.Errorf("none of the configured tables %v were found in any schema matching pattern %q", config.DBTables, config.DBSchemaPattern) + // A table must exist in at least one matched schema. Missing from some + // (but not all) matched schemas is tolerated above as a multi-tenant gap; + // missing from every matched schema is indistinguishable from a typo and + // must fail loudly rather than silently drop the table. + var missingTables []string + for i, table := range normalizedTables { + if !foundTables[table] { + missingTables = append(missingTables, config.DBTables[i]) + } + } + if len(missingTables) > 0 { + return nil, fmt.Errorf("table(s) %v not found in any schema matching pattern %q", missingTables, config.DBSchemaPattern) } batchSize := 1000 if config.BatchSize > 0 { From c9e488b42dc50b324627a702c3a63b64942c6242 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Thu, 6 Aug 2026 14:06:05 +0100 Subject: [PATCH 23/48] update docs --- docs/modules/components/pages/inputs/postgres_cdc.adoc | 4 +++- internal/impl/postgresql/input_pg_stream.go | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 849adbd8ae..e7f995f242 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -178,6 +178,8 @@ Double-quoted identifiers are treated as exact names and do not support wildcard Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted. +If `tables` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when `tables` is left empty - see `tables` below. + *Type*: `string` @@ -198,7 +200,7 @@ schema: '*' A list of table names to include in the logical replication. Each table should be specified as a separate item. -When `schema` is a glob pattern, this list is resolved against each matched schema independently: a table missing from a given schema is skipped (with a warning logged) rather than failing replication for every matched schema. +When `schema` is a glob pattern, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. If left empty, the underlying PostgreSQL publication is created `FOR ALL TABLES`, which replicates every table in every schema of the database, ignoring `schema`. This also disables `stream_snapshot`, since the initial snapshot is only planned for tables listed here. diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 5cecc17457..5598c0b24f 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -122,13 +122,15 @@ When a pattern is used, all schemas whose names match the pattern are replicated Double-quoted identifiers are treated as exact names and do not support wildcards. -Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted.`). +Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted. + +If ` + "`" + fieldTables + "`" + ` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when ` + "`" + fieldTables + "`" + ` is left empty - see ` + "`" + fieldTables + "`" + ` below.`). Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`, "tenant_*", "*"), ). Field(service.NewStringListField(fieldTables). Description(`A list of table names to include in the logical replication. Each table should be specified as a separate item. -When ` + "`schema`" + ` is a glob pattern, this list is resolved against each matched schema independently: a table missing from a given schema is skipped (with a warning logged) rather than failing replication for every matched schema. +When ` + "`schema`" + ` is a glob pattern, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. If left empty, the underlying PostgreSQL publication is created ` + "`FOR ALL TABLES`" + `, which replicates every table in every schema of the database, ignoring ` + "`" + fieldSchema + "`" + `. This also disables ` + "`" + fieldStreamSnapshot + "`" + `, since the initial snapshot is only planned for tables listed here.`). Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`})). From 0cd24bdc7ec2a51e6b944a3dd8bcedda8de3f3e0 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Tue, 11 Aug 2026 01:23:56 +0300 Subject: [PATCH 24/48] postgres_cdc: add schema_pattern field for multi-schema replication Introduces schema_pattern as a new optional field for glob-based multi-schema CDC, instead of overloading schema with pattern semantics. This keeps schema backwards compatible: it reverts to exact-name-only behavior (now defaulting to "public"), while schema_pattern opts into the glob resolution path. Setting both is a config error. Co-Authored-By: Claude Sonnet 5 --- .../components/pages/inputs/postgres_cdc.adoc | 38 +++++-- internal/impl/postgresql/input_pg_stream.go | 46 +++++--- .../impl/postgresql/input_pg_stream_test.go | 77 ++++++++++--- internal/impl/postgresql/integration_test.go | 14 +-- .../impl/postgresql/pglogicalstream/config.go | 7 +- .../pglogicalstream/logical_stream.go | 101 ++++++++++-------- 6 files changed, 197 insertions(+), 86 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index e7f995f242..86702205a5 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -43,7 +43,8 @@ input: include_transaction_markers: false stream_snapshot: false snapshot_batch_size: 1000 - schema: public # No default (required) + schema: public + schema_pattern: "" tables: [] # No default (required) checkpoint_limit: 1024 temporary_slot: false @@ -73,7 +74,8 @@ input: include_transaction_markers: false stream_snapshot: false snapshot_batch_size: 1000 - schema: public # No default (required) + schema: public + schema_pattern: "" tables: [] # No default (required) checkpoint_limit: 1024 temporary_slot: false @@ -170,7 +172,24 @@ snapshot_batch_size: 10000 === `schema` -The PostgreSQL schema to replicate data from. Accepts an exact schema name or a glob pattern using `*` as a wildcard to match multiple schemas. +The PostgreSQL schema from which to replicate data. + + +*Type*: `string` + +*Default*: `"public"` + +```yml +# Examples + +schema: public + +schema: '"MyCaseSensitiveSchemaNeedingQuotes"' +``` + +=== `schema_pattern` + +The PostgreSQL schema pattern to replicate data from. Accepts an exact schema name or a glob pattern using `*` as a wildcard to match multiple schemas. When a pattern is used, all schemas whose names match the pattern are replicated using a single replication slot and publication. This is useful for multi-tenant databases where each tenant has its own schema (e.g. `tenant_*` matches `tenant_foo`, `tenant_bar`, etc.). @@ -180,27 +199,28 @@ Schema pattern matching runs once at pipeline startup. Schemas created after the If `tables` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when `tables` is left empty - see `tables` below. +This field is mutually exclusive with `schema`; when set, it takes over schema resolution entirely and `schema` must be left at its default. + *Type*: `string` +*Default*: `""` ```yml # Examples -schema: public - -schema: '"MyCaseSensitiveSchemaNeedingQuotes"' +schema_pattern: tenant_* -schema: tenant_* +schema_pattern: '*' -schema: '*' +schema_pattern: '"MyCaseSensitiveSchemaNeedingQuotes"' ``` === `tables` A list of table names to include in the logical replication. Each table should be specified as a separate item. -When `schema` is a glob pattern, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. +When `schema_pattern` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. If left empty, the underlying PostgreSQL publication is created `FOR ALL TABLES`, which replicates every table in every schema of the database, ignoring `schema`. This also disables `stream_snapshot`, since the initial snapshot is only planned for tables listed here. diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 5598c0b24f..8dbb87f5d0 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -38,6 +38,7 @@ const ( fieldSnapshotMemSafetyFactor = "snapshot_memory_safety_factor" fieldSnapshotBatchSize = "snapshot_batch_size" fieldSchema = "schema" + fieldSchemaPattern = "schema_pattern" fieldTables = "tables" fieldCheckpointLimit = "checkpoint_limit" fieldTemporarySlot = "temporary_slot" @@ -116,7 +117,13 @@ This input adds the following metadata fields to each message: Example(10000). Default(1000)). Field(service.NewStringField(fieldSchema). - Description(`The PostgreSQL schema to replicate data from. Accepts an exact schema name or a glob pattern using `+"`*`"+` as a wildcard to match multiple schemas. + Description("The PostgreSQL schema from which to replicate data."). + Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`). + Optional(). + Default("public"), + ). + Field(service.NewStringField(fieldSchemaPattern). + Description(`The PostgreSQL schema pattern to replicate data from. Accepts an exact schema name or a glob pattern using `+"`*`"+` as a wildcard to match multiple schemas. When a pattern is used, all schemas whose names match the pattern are replicated using a single replication slot and publication. This is useful for multi-tenant databases where each tenant has its own schema (e.g. `+"`tenant_*`"+` matches `+"`tenant_foo`"+`, `+"`tenant_bar`"+`, etc.). @@ -124,13 +131,17 @@ Double-quoted identifiers are treated as exact names and do not support wildcard Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted. -If ` + "`" + fieldTables + "`" + ` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when ` + "`" + fieldTables + "`" + ` is left empty - see ` + "`" + fieldTables + "`" + ` below.`). - Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`, "tenant_*", "*"), +If `+"`"+fieldTables+"`"+` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when `+"`"+fieldTables+"`"+` is left empty - see `+"`"+fieldTables+"`"+` below. + +This field is mutually exclusive with `+"`"+fieldSchema+"`"+`; when set, it takes over schema resolution entirely and `+"`"+fieldSchema+"`"+` must be left at its default.`). + Examples("tenant_*", "*", `"MyCaseSensitiveSchemaNeedingQuotes"`). + Optional(). + Default(""), ). Field(service.NewStringListField(fieldTables). Description(`A list of table names to include in the logical replication. Each table should be specified as a separate item. -When ` + "`schema`" + ` is a glob pattern, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. +When ` + "`schema_pattern`" + ` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. If left empty, the underlying PostgreSQL publication is created ` + "`FOR ALL TABLES`" + `, which replicates every table in every schema of the database, ignoring ` + "`" + fieldSchema + "`" + `. This also disables ` + "`" + fieldStreamSnapshot + "`" + `, since the initial snapshot is only planned for tables listed here.`). Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`})). @@ -231,6 +242,7 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser dbSlotName string temporarySlot bool schema string + schemaPattern string tables []string streamSnapshot bool includeTxnMarkers bool @@ -275,14 +287,23 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser if schema, err = conf.FieldString(fieldSchema); err != nil { return nil, err } - if err = validateSchemaPattern(schema); err != nil { - return nil, fmt.Errorf("invalid schema: %w", err) + + if schemaPattern, err = conf.FieldString(fieldSchemaPattern); err != nil { + return nil, err } - // Normalize unquoted patterns to lower-case: PostgreSQL folds unquoted - // identifiers at creation time, so TENANT_* and tenant_* resolve identically. - // Normalizing early avoids silent case-folding surprises in resolveSchemas. - if !strings.HasPrefix(schema, `"`) { - schema = strings.ToLower(schema) + if schemaPattern != "" { + if schema != "public" { + return nil, errors.New("schema and schema_pattern are mutually exclusive") + } + if err = validateSchemaPattern(schemaPattern); err != nil { + return nil, fmt.Errorf("invalid schema_pattern: %w", err) + } + // Normalize unquoted patterns to lower-case: PostgreSQL folds unquoted + // identifiers at creation time, so TENANT_* and tenant_* resolve identically. + // Normalizing early avoids silent case-folding surprises in resolveSchemas. + if !strings.HasPrefix(schemaPattern, `"`) { + schemaPattern = strings.ToLower(schemaPattern) + } } if tables, err = conf.FieldStringList(fieldTables); err != nil { @@ -363,7 +384,8 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser DBConfig: pgConnConfig, TLSConfig: pgConnConfig.TLSConfig, DBRawDSN: dsn, - DBSchemaPattern: schema, + DBSchema: schema, + DBSchemaPattern: schemaPattern, DBTables: tables, RefreshAuthToken: iamAuthTokenBuilder, diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index 19d9723bc2..251a963aa8 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -20,16 +20,40 @@ import ( "github.com/redpanda-data/connect/v4/internal/license" ) -// TestSchemaPatternValidation verifies that the schema field is validated -// during config parsing, before any network I/O is attempted. Success is -// asserted via newPgStreamInput returning no error - the constructor doesn't -// dial the database, so a valid pattern implies validation passed. +func parsePgStreamInput(t *testing.T, yaml string) (service.BatchInput, error) { + t.Helper() + conf, err := newPostgresCDCConfig().ParseYAML(yaml, nil) + require.NoError(t, err) + + mgr := service.MockResources() + license.InjectTestService(mgr) + + return newPgStreamInput(conf, mgr) +} + +// TestSchemaDefault verifies that the schema field defaults to "public" when +// left unset, matching pre-multi-schema behaviour. +func TestSchemaDefault(t *testing.T) { + yaml := ` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +slot_name: test_slot +tables: + - events +` + _, err := parsePgStreamInput(t, yaml) + require.NoError(t, err) +} + +// TestSchemaPatternValidation verifies that the schema_pattern field is +// validated during config parsing, before any network I/O is attempted. +// Success is asserted via newPgStreamInput returning no error - the +// constructor doesn't dial the database, so a valid pattern implies +// validation passed. func TestSchemaPatternValidation(t *testing.T) { tests := []struct { pattern string errContains string }{ - {"public", ""}, {"tenant_*", ""}, {"*", ""}, {`"MySchema"`, ""}, @@ -39,7 +63,6 @@ func TestSchemaPatternValidation(t *testing.T) { {"1abc", "must start with a letter"}, {`"unclosed`, "invalid quoted schema identifier"}, {"schema-name", "invalid character"}, - {"", "schema cannot be empty"}, {`"quoted*"`, "wildcard"}, } for _, tt := range tests { @@ -48,19 +71,13 @@ func TestSchemaPatternValidation(t *testing.T) { // quotes, e.g. `"MySchema"`) reaches validateSchemaPattern verbatim. yaml := fmt.Sprintf(` dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable -schema: '%s' +schema_pattern: '%s' slot_name: test_slot tables: - events `, tt.pattern) - conf, err := newPostgresCDCConfig().ParseYAML(yaml, nil) - require.NoError(t, err) - - mgr := service.MockResources() - license.InjectTestService(mgr) - - _, err = newPgStreamInput(conf, mgr) + _, err := parsePgStreamInput(t, yaml) if tt.errContains != "" { require.Error(t, err) assert.Contains(t, err.Error(), tt.errContains) @@ -70,3 +87,35 @@ tables: }) } } + +// TestSchemaAndSchemaPatternMutuallyExclusive verifies that setting both +// schema (to a non-default value) and schema_pattern is rejected at config +// construction time. +func TestSchemaAndSchemaPatternMutuallyExclusive(t *testing.T) { + yaml := ` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +schema: tenant_foo +schema_pattern: 'tenant_*' +slot_name: test_slot +tables: + - events +` + _, err := parsePgStreamInput(t, yaml) + require.Error(t, err) + assert.Contains(t, err.Error(), "schema and schema_pattern are mutually exclusive") +} + +// TestSchemaPatternWithDefaultSchemaSucceeds verifies that setting +// schema_pattern while leaving schema untouched (at its "public" default) is +// allowed. +func TestSchemaPatternWithDefaultSchemaSucceeds(t *testing.T) { + yaml := ` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +schema_pattern: 'tenant_*' +slot_name: test_slot +tables: + - events +` + _, err := parsePgStreamInput(t, yaml) + require.NoError(t, err) +} diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 35adb4cc28..acb5458439 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1672,7 +1672,7 @@ postgres_cdc: dsn: %s slot_name: multi_schema_test_slot stream_snapshot: true - schema: tenant_* + schema_pattern: tenant_* tables: - events `, databaseURL) @@ -1795,7 +1795,7 @@ postgres_cdc: dsn: %s slot_name: missing_table_degrade_slot stream_snapshot: true - schema: tenant_* + schema_pattern: tenant_* tables: - events `, databaseURL) @@ -1847,7 +1847,7 @@ func TestIntegrationNoSchemasMatchedReturnsError(t *testing.T) { tmpl := fmt.Sprintf(` dsn: %s slot_name: no_schema_match_slot -schema: nonexistent_schema_zzz_* +schema_pattern: nonexistent_schema_zzz_* tables: - events `, databaseURL) @@ -1886,7 +1886,7 @@ func TestIntegrationForAllTablesIgnoresNonMatchingSchemaPattern(t *testing.T) { postgres_cdc: dsn: %s slot_name: for_all_tables_ignores_schema_slot - schema: nonexistent_schema_zzz_* + schema_pattern: nonexistent_schema_zzz_* `, databaseURL) // FOR ALL TABLES mode disables the initial snapshot, so replication only @@ -1994,7 +1994,7 @@ tables: tmpl := fmt.Sprintf(` dsn: %s slot_name: glob_schema_total_miss_slot -schema: tenant_* +schema_pattern: tenant_* tables: - orders - ordres @@ -2048,7 +2048,7 @@ postgres_cdc: dsn: %s slot_name: glob_schema_partial_match_slot stream_snapshot: true - schema: tenant_* + schema_pattern: tenant_* tables: - orders - ordres @@ -2127,7 +2127,7 @@ postgres_cdc: dsn: %s slot_name: glob_schema_full_match_slot stream_snapshot: true - schema: tenant_* + schema_pattern: tenant_* tables: - orders - ordres diff --git a/internal/impl/postgresql/pglogicalstream/config.go b/internal/impl/postgresql/pglogicalstream/config.go index 8fed67862e..f6cb5ce99c 100644 --- a/internal/impl/postgresql/pglogicalstream/config.go +++ b/internal/impl/postgresql/pglogicalstream/config.go @@ -24,8 +24,11 @@ type Config struct { DBConfig *pgconn.Config DBRawDSN string TLSConfig *tls.Config - // DBSchemaPattern is the schema to replicate from. Accepts an exact schema - // name or a glob pattern using '*' as a wildcard (e.g. "tenant_*", "*"). + DBSchema string + // DBSchemaPattern is the glob pattern used to replicate from multiple + // schemas at once, using '*' as a wildcard (e.g. "tenant_*", "*"). When + // non-empty, it takes precedence over DBSchema and schemas are resolved + // dynamically at stream creation time. DBSchemaPattern string DBTables []string // Refreshes short lived IAM auth token that is treated as a password diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 5747803116..669acbc66a 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -98,55 +98,72 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { return nil, err } - schemas, inaccessibleSchemas, err := resolveSchemas(ctx, dbConn, config.DBSchemaPattern) - if err != nil { - return nil, fmt.Errorf("resolving schema pattern %q: %w", config.DBSchemaPattern, err) - } - if len(inaccessibleSchemas) > 0 && len(config.DBTables) > 0 { - config.Logger.Warnf("schema pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaPattern, inaccessibleSchemas) - } - if len(schemas) == 0 && len(config.DBTables) > 0 { - return nil, fmt.Errorf("no schemas found matching pattern %q", config.DBSchemaPattern) - } - config.Logger.Infof("Schema pattern %q resolved to %d schema(s): %v", config.DBSchemaPattern, len(schemas), schemas) - - normalizedTables := make([]string, 0, len(config.DBTables)) - for _, table := range config.DBTables { - normalized, err := sanitize.NormalizePostgresIdentifier(table) + var tables []TableFQN + if config.DBSchemaPattern != "" { + schemas, inaccessibleSchemas, err := resolveSchemas(ctx, dbConn, config.DBSchemaPattern) if err != nil { - return nil, fmt.Errorf("invalid table name %q: %w", table, err) + return nil, fmt.Errorf("resolving schema pattern %q: %w", config.DBSchemaPattern, err) } - normalizedTables = append(normalizedTables, normalized) - } + if len(inaccessibleSchemas) > 0 && len(config.DBTables) > 0 { + config.Logger.Warnf("schema pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaPattern, inaccessibleSchemas) + } + if len(schemas) == 0 && len(config.DBTables) > 0 { + return nil, fmt.Errorf("no schemas found matching pattern %q", config.DBSchemaPattern) + } + config.Logger.Infof("Schema pattern %q resolved to %d schema(s): %v", config.DBSchemaPattern, len(schemas), schemas) - tables := make([]TableFQN, 0, len(schemas)*len(normalizedTables)) - foundTables := make(map[string]bool, len(normalizedTables)) - for _, schema := range schemas { - existingTables, err := resolveExistingTables(ctx, dbConn, schema) - if err != nil { - return nil, fmt.Errorf("resolving tables in schema %q: %w", schema, err) + normalizedTables := make([]string, 0, len(config.DBTables)) + for _, table := range config.DBTables { + normalized, err := sanitize.NormalizePostgresIdentifier(table) + if err != nil { + return nil, fmt.Errorf("invalid table name %q: %w", table, err) + } + normalizedTables = append(normalizedTables, normalized) } - for _, table := range normalizedTables { - if _, ok := existingTables[table]; !ok { - config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched pattern %q but does not contain this table)", schema, table, schema, config.DBSchemaPattern) - continue + + tables = make([]TableFQN, 0, len(schemas)*len(normalizedTables)) + foundTables := make(map[string]bool, len(normalizedTables)) + for _, schema := range schemas { + existingTables, err := resolveExistingTables(ctx, dbConn, schema) + if err != nil { + return nil, fmt.Errorf("resolving tables in schema %q: %w", schema, err) + } + for _, table := range normalizedTables { + if _, ok := existingTables[table]; !ok { + config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched pattern %q but does not contain this table)", schema, table, schema, config.DBSchemaPattern) + continue + } + tables = append(tables, TableFQN{Schema: schema, Table: table}) + foundTables[table] = true } - tables = append(tables, TableFQN{Schema: schema, Table: table}) - foundTables[table] = true } - } - // A table must exist in at least one matched schema. Missing from some - // (but not all) matched schemas is tolerated above as a multi-tenant gap; - // missing from every matched schema is indistinguishable from a typo and - // must fail loudly rather than silently drop the table. - var missingTables []string - for i, table := range normalizedTables { - if !foundTables[table] { - missingTables = append(missingTables, config.DBTables[i]) + // A table must exist in at least one matched schema. Missing from some + // (but not all) matched schemas is tolerated above as a multi-tenant gap; + // missing from every matched schema is indistinguishable from a typo and + // must fail loudly rather than silently drop the table. + var missingTables []string + for i, table := range normalizedTables { + if !foundTables[table] { + missingTables = append(missingTables, config.DBTables[i]) + } + } + if len(missingTables) > 0 { + return nil, fmt.Errorf("table(s) %v not found in any schema matching pattern %q", missingTables, config.DBSchemaPattern) + } + } else { + schema, err := sanitize.NormalizePostgresIdentifier(config.DBSchema) + if err != nil { + return nil, fmt.Errorf("invalid schema name %q: %w", config.DBSchema, err) + } + + tables = []TableFQN{} + for _, table := range config.DBTables { + normalized, err := sanitize.NormalizePostgresIdentifier(table) + if err != nil { + return nil, fmt.Errorf("invalid table name %q: %w", table, err) + } + tables = append(tables, TableFQN{Schema: schema, Table: normalized}) } - } - if len(missingTables) > 0 { - return nil, fmt.Errorf("table(s) %v not found in any schema matching pattern %q", missingTables, config.DBSchemaPattern) } batchSize := 1000 if config.BatchSize > 0 { From 33e4049f6c4412252f843138d7bf9c30a8550bd8 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Tue, 11 Aug 2026 10:40:06 +0300 Subject: [PATCH 25/48] postgres_cdc: fix unicode schema_pattern rejection, sync changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validateSchemaPattern only accepted [a-zA-Z0-9_*], stricter than sanitize.NormalizePostgresIdentifier (unicode.IsLetter/IsDigit) which governs the exact-name schema path. Since schema_pattern is now the sole caller of validateSchemaPattern, an unquoted pattern with non-ASCII letters (e.g. münchen) was rejected at startup even though the equivalent schema value has always been accepted. Widened the character classes to match, using utf8.DecodeRuneInString for the first-character check instead of a raw byte cast. Also corrected the Unreleased changelog entry, which still described the old design where schema itself accepted glob patterns. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 2 +- internal/impl/postgresql/input_pg_stream.go | 8 ++++--- .../impl/postgresql/input_pg_stream_test.go | 23 +++++++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51e5da7716..006280950c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,7 +69,7 @@ All notable changes to this project will be documented in this file. ### Added -- postgres_cdc: Postgres CDC now accepts a glob pattern for the `schema` field (e.g. `tenant_*`), replicating all matching schemas through a single replication slot. Useful for multi-tenant databases where each tenant has its own schema. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) +- postgres_cdc: Added a new `schema_pattern` field accepting a glob pattern (e.g. `tenant_*`), replicating all matching schemas through a single replication slot. Useful for multi-tenant databases where each tenant has its own schema. The existing `schema` field is unaffected and continues to take a single exact schema name (defaulting to `public`); `schema` and `schema_pattern` are mutually exclusive. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) - aws_dynamodb_cdc: DynamoDB CDC now supports an optional checkpoint_namespace field, allowing multiple independent pipelines to share a single checkpoint table without overwriting each other's checkpoints. ([@squiidz](https://github.com/squiidz), [#4602](https://github.com/redpanda-data/connect/pull/4602)) ### Fixed diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 8dbb87f5d0..d6d7235fd0 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -18,6 +18,8 @@ import ( "strings" "sync" "time" + "unicode" + "unicode/utf8" "github.com/Jeffail/checkpoint" "github.com/Jeffail/shutdown" @@ -444,13 +446,13 @@ func validateSchemaPattern(s string) error { return nil } for i, ch := range s { - if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '*' { + if unicode.IsLetter(ch) || unicode.IsDigit(ch) || ch == '_' || ch == '*' { continue } return fmt.Errorf("invalid character %q at position %d in schema pattern %q", ch, i, s) } - first := rune(s[0]) - if first != '_' && first != '*' && (first < 'a' || first > 'z') && (first < 'A' || first > 'Z') { + first, _ := utf8.DecodeRuneInString(s) + if first != '_' && first != '*' && !unicode.IsLetter(first) { return fmt.Errorf("schema pattern %q must start with a letter, underscore, or '*'", s) } return nil diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index 251a963aa8..2b30df9fa2 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -57,6 +57,11 @@ func TestSchemaPatternValidation(t *testing.T) { {"tenant_*", ""}, {"*", ""}, {`"MySchema"`, ""}, + // Regression test: validateSchemaPattern must accept the same unicode + // letters/digits that sanitize.NormalizePostgresIdentifier accepts for + // unquoted identifiers (e.g. "münchen"), not just ASCII. + {"münchen", ""}, + {"tenant_ü*", ""}, // Regression test: len("") == 2 used to pass the old `len(s) < 2` guard. // Fixed to `len(s) < 3`. {`""`, "invalid quoted schema identifier"}, @@ -119,3 +124,21 @@ tables: _, err := parsePgStreamInput(t, yaml) require.NoError(t, err) } + +// TestSchemaAcceptsUnicodeIdentifier verifies that the schema field (single +// exact-name path) still accepts unquoted unicode identifiers like +// "münchen", matching sanitize.NormalizePostgresIdentifier which is the sole +// validator on this path (see NewPgStream). schema no longer runs through +// validateSchemaPattern, so this guards against that ASCII-only validator +// regressing this path again in the future. +func TestSchemaAcceptsUnicodeIdentifier(t *testing.T) { + yaml := ` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +schema: münchen +slot_name: test_slot +tables: + - events +` + _, err := parsePgStreamInput(t, yaml) + require.NoError(t, err) +} From b428bd7af2b4cde5023cf4c3ef473e48903865fe Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Tue, 11 Aug 2026 15:52:17 +0300 Subject: [PATCH 26/48] postgres_cdc: fix stale schema_pattern refs in manual test harness internal/impl/postgresql/tests/current/ is a docker-compose-backed Taskfile harness for manually exercising multi-schema CDC against a real Postgres (tenant_a/tenant_b). It predates the schema/schema_pattern split and still used schema: tenant_* directly, which the split turns into a literal (and invalid) exact schema name instead of a glob. - test_config.yaml: schema -> schema_pattern for the tenant_* glob. - Taskfile.yaml test:invalid-schema: schema="" is now a no-op (unset schema_pattern falls back to schema's "public" default), so it no longer exercises a failure path. Repointed at schema_pattern=1abc, which still fails fast in newPgStreamInput before any DB connection, matching the task's original intent. Verified by bringing the compose stack up and running the smoke test; config loads and reaches runtime init (blocked only by the license check in this sandboxed environment, which is unrelated). Co-Authored-By: Claude Sonnet 5 --- internal/impl/postgresql/tests/current/Taskfile.yaml | 11 ++++++++--- .../impl/postgresql/tests/current/test_config.yaml | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/internal/impl/postgresql/tests/current/Taskfile.yaml b/internal/impl/postgresql/tests/current/Taskfile.yaml index 75551c4d34..d81e5413ce 100644 --- a/internal/impl/postgresql/tests/current/Taskfile.yaml +++ b/internal/impl/postgresql/tests/current/Taskfile.yaml @@ -72,17 +72,22 @@ tasks: # ── Schema validation smoke test ────────────────────────────────────────────── test:invalid-schema: - desc: Confirm that an empty quoted schema identifier is rejected at startup + desc: Confirm that a malformed schema_pattern is rejected at startup, before any DB connection is attempted env: PG_DSN: '{{.PG_DSN}}' cmds: - | set +e go run ../../../../../cmd/redpanda-connect/main.go run \ - --set 'input.postgres_cdc.schema=""' \ + --set 'input.postgres_cdc.schema_pattern=1abc' \ ./test_config.yaml 2>&1 | head -5 echo "exit $?" - # Expects: "invalid schema" error printed, process exits non-zero. + # Expects: "invalid schema_pattern" error printed, process exits non-zero. + # schema_pattern is validated eagerly in newPgStreamInput + # (validateSchemaPattern), unlike schema which is only checked later, + # against the live DB connection, inside NewPgStream. + # Note: an empty schema_pattern is NOT an error - it means "unset", and + # the connector falls back to the schema field (default "public"). # ── Quick sanity ────────────────────────────────────────────────────────────── diff --git a/internal/impl/postgresql/tests/current/test_config.yaml b/internal/impl/postgresql/tests/current/test_config.yaml index 9e40b5a1c1..838321a392 100644 --- a/internal/impl/postgresql/tests/current/test_config.yaml +++ b/internal/impl/postgresql/tests/current/test_config.yaml @@ -4,7 +4,7 @@ input: slot_name: multi_schema_test_slot stream_snapshot: true # Glob pattern: replicates both tenant_a and tenant_b via one slot. - schema: tenant_* + schema_pattern: tenant_* tables: - events From b46a4e5af60767b223be62e99c9679d25f85afe8 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Tue, 11 Aug 2026 21:59:54 +0300 Subject: [PATCH 27/48] postgres_cdc: add excluded_schema --- CHANGELOG.md | 1 + .../components/pages/inputs/postgres_cdc.adoc | 23 +++ internal/impl/postgresql/input_pg_stream.go | 33 ++++ .../impl/postgresql/input_pg_stream_test.go | 73 +++++++++ internal/impl/postgresql/integration_test.go | 152 ++++++++++++++++++ .../impl/postgresql/pglogicalstream/config.go | 6 +- .../pglogicalstream/logical_stream.go | 30 ++++ .../pglogicalstream/schema_resolver.go | 51 ++++++ .../pglogicalstream/schema_resolver_test.go | 40 +++++ .../postgresql/tests/current/Taskfile.yaml | 19 ++- .../impl/postgresql/tests/current/setup.sql | 18 ++- .../postgresql/tests/current/test_config.yaml | 4 + .../test_config_exclude_no_pattern.yaml | 22 +++ 13 files changed, 469 insertions(+), 3 deletions(-) create mode 100644 internal/impl/postgresql/tests/current/test_config_exclude_no_pattern.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 006280950c..857a23158d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,7 @@ All notable changes to this project will be documented in this file. ### Added - postgres_cdc: Added a new `schema_pattern` field accepting a glob pattern (e.g. `tenant_*`), replicating all matching schemas through a single replication slot. Useful for multi-tenant databases where each tenant has its own schema. The existing `schema` field is unaffected and continues to take a single exact schema name (defaulting to `public`); `schema` and `schema_pattern` are mutually exclusive. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) +- postgres_cdc: Added a new `exclude_schemas` field to carve exceptions out of a broad `schema_pattern` (e.g. `schema_pattern: tenant_*` while skipping `tenant_test`). Accepts the same exact-name/glob/quoted syntax as `schema_pattern`, matches entries against the already-resolved schema list in memory with no extra database round-trips, and requires `schema_pattern` to be set. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) - aws_dynamodb_cdc: DynamoDB CDC now supports an optional checkpoint_namespace field, allowing multiple independent pipelines to share a single checkpoint table without overwriting each other's checkpoints. ([@squiidz](https://github.com/squiidz), [#4602](https://github.com/redpanda-data/connect/pull/4602)) ### Fixed diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 86702205a5..6332e8fd3c 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -45,6 +45,7 @@ input: snapshot_batch_size: 1000 schema: public schema_pattern: "" + exclude_schemas: [] tables: [] # No default (required) checkpoint_limit: 1024 temporary_slot: false @@ -76,6 +77,7 @@ input: snapshot_batch_size: 1000 schema: public schema_pattern: "" + exclude_schemas: [] tables: [] # No default (required) checkpoint_limit: 1024 temporary_slot: false @@ -216,6 +218,27 @@ schema_pattern: '*' schema_pattern: '"MyCaseSensitiveSchemaNeedingQuotes"' ``` +=== `exclude_schemas` + +A list of schema names or glob patterns to exclude from the schemas matched by `schema_pattern`. Only valid when `schema_pattern` is set. + +Each entry uses the same syntax as `schema_pattern`: an exact schema name, a glob pattern using `*` as a wildcard, or a double-quoted exact identifier for an exact, case-sensitive match. + +A schema that matches `schema_pattern` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by `schema_pattern` is silently ignored, so a typo here simply excludes nothing rather than failing startup. + + +*Type*: `array` + +*Default*: `[]` + +```yml +# Examples + +exclude_schemas: + - tenant_internal + - tenant_test_* +``` + === `tables` A list of table names to include in the logical replication. Each table should be specified as a separate item. diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index d6d7235fd0..17959f19e1 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -41,6 +41,7 @@ const ( fieldSnapshotBatchSize = "snapshot_batch_size" fieldSchema = "schema" fieldSchemaPattern = "schema_pattern" + fieldExcludeSchemas = "exclude_schemas" fieldTables = "tables" fieldCheckpointLimit = "checkpoint_limit" fieldTemporarySlot = "temporary_slot" @@ -140,6 +141,16 @@ This field is mutually exclusive with `+"`"+fieldSchema+"`"+`; when set, it take Optional(). Default(""), ). + Field(service.NewStringListField(fieldExcludeSchemas). + Description(`A list of schema names or glob patterns to exclude from the schemas matched by ` + "`" + fieldSchemaPattern + "`" + `. Only valid when ` + "`" + fieldSchemaPattern + "`" + ` is set. + +Each entry uses the same syntax as ` + "`" + fieldSchemaPattern + "`" + `: an exact schema name, a glob pattern using ` + "`*`" + ` as a wildcard, or a double-quoted exact identifier for an exact, case-sensitive match. + +A schema that matches ` + "`" + fieldSchemaPattern + "`" + ` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by ` + "`" + fieldSchemaPattern + "`" + ` is silently ignored, so a typo here simply excludes nothing rather than failing startup.`). + Examples([]string{"tenant_internal", "tenant_test_*"}). + Optional(). + Default([]string{}), + ). Field(service.NewStringListField(fieldTables). Description(`A list of table names to include in the logical replication. Each table should be specified as a separate item. @@ -245,6 +256,7 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser temporarySlot bool schema string schemaPattern string + excludeSchemas []string tables []string streamSnapshot bool includeTxnMarkers bool @@ -308,6 +320,26 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser } } + if excludeSchemas, err = conf.FieldStringList(fieldExcludeSchemas); err != nil { + return nil, err + } + if len(excludeSchemas) > 0 { + if schemaPattern == "" { + return nil, errors.New("exclude_schemas requires schema_pattern to be set") + } + for i, pattern := range excludeSchemas { + if err = validateSchemaPattern(pattern); err != nil { + return nil, fmt.Errorf("invalid exclude_schemas entry %q: %w", pattern, err) + } + // Normalize unquoted patterns to lower-case, mirroring schema_pattern + // above: PostgreSQL folds unquoted identifiers at creation time, so + // TENANT_TEST and tenant_test resolve to the same schema. + if !strings.HasPrefix(pattern, `"`) { + excludeSchemas[i] = strings.ToLower(pattern) + } + } + } + if tables, err = conf.FieldStringList(fieldTables); err != nil { return nil, err } @@ -388,6 +420,7 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser DBRawDSN: dsn, DBSchema: schema, DBSchemaPattern: schemaPattern, + DBExcludeSchemas: excludeSchemas, DBTables: tables, RefreshAuthToken: iamAuthTokenBuilder, diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index 2b30df9fa2..d7869eaa4f 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -125,6 +125,79 @@ tables: require.NoError(t, err) } +// TestExcludeSchemasValidation verifies that each exclude_schemas entry is +// validated with the same rules as schema_pattern - validateSchemaPattern is +// reused rather than re-derived, so this exercises the same error cases +// TestSchemaPatternValidation covers, just reached through a different field. +func TestExcludeSchemasValidation(t *testing.T) { + tests := []struct { + pattern string + errContains string + }{ + {"tenant_test", ""}, + {"tenant_test_*", ""}, + {`"MySchema"`, ""}, + {"1abc", "must start with a letter"}, + {`"unclosed`, "invalid quoted schema identifier"}, + {"schema-name", "invalid character"}, + {`"quoted*"`, "wildcard"}, + } + for _, tt := range tests { + t.Run(tt.pattern, func(t *testing.T) { + // Single-quoted so the pattern (which may itself contain double + // quotes, e.g. `"MySchema"`) reaches validateSchemaPattern verbatim. + yaml := fmt.Sprintf(` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +schema_pattern: 'tenant_*' +exclude_schemas: ['%s'] +slot_name: test_slot +tables: + - events +`, tt.pattern) + + _, err := parsePgStreamInput(t, yaml) + if tt.errContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errContains) + return + } + require.NoError(t, err) + }) + } +} + +// TestExcludeSchemasRequiresSchemaPattern verifies that exclude_schemas is +// rejected at config-parse time when schema_pattern is left unset. Both +// single-exact-schema mode and FOR ALL TABLES mode (empty tables) have no +// well-defined candidate set to exclude from, so this is a hard error rather +// than a silent no-op. +func TestExcludeSchemasRequiresSchemaPattern(t *testing.T) { + yaml := ` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +exclude_schemas: [tenant_test] +slot_name: test_slot +tables: + - events +` + _, err := parsePgStreamInput(t, yaml) + require.Error(t, err) + assert.Contains(t, err.Error(), "exclude_schemas requires schema_pattern to be set") +} + +// TestExcludeSchemasEmptyWithoutSchemaPatternSucceeds verifies that leaving +// exclude_schemas at its default empty list does not trip the +// requires-schema_pattern check, since there's nothing to exclude. +func TestExcludeSchemasEmptyWithoutSchemaPatternSucceeds(t *testing.T) { + yaml := ` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +slot_name: test_slot +tables: + - events +` + _, err := parsePgStreamInput(t, yaml) + require.NoError(t, err) +} + // TestSchemaAcceptsUnicodeIdentifier verifies that the schema field (single // exact-name path) still accepts unquoted unicode identifiers like // "münchen", matching sanitize.NormalizePostgresIdentifier which is the sole diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index acb5458439..1b77789324 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1760,6 +1760,158 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } +// TestIntegrationMultiSchemaExcludeSchemas verifies that exclude_schemas +// carves an exception out of a broad schema_pattern: a schema that matches +// schema_pattern but also matches an exclude_schemas entry contributes no +// rows at all, neither during the initial snapshot nor from subsequent CDC +// changes. +func TestIntegrationMultiSchemaExcludeSchemas(t *testing.T) { + integration.CheckSkip(t) + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + // Three tenant schemas match tenant_*; tenant_c is carved out via exclude_schemas. + for _, schema := range []string{"tenant_a", "tenant_b", "tenant_c"} { + _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", schema)) + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf( + "CREATE TABLE %s.events (id SERIAL PRIMARY KEY, name TEXT)", schema)) + require.NoError(t, err) + } + + // Pre-load snapshot data, including a row in the excluded schema that must + // never surface. + _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('alice'), ('bob')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_b.events (name) VALUES ('carol')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_c.events (name) VALUES ('mallory')") + require.NoError(t, err) + + type msgMeta struct { + dbSchema string + table string + operation string + lsn string + } + + var ( + mu sync.Mutex + collected []msgMeta + ) + collectedLen := func() int { + mu.Lock() + defer mu.Unlock() + return len(collected) + } + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: exclude_schemas_test_slot + stream_snapshot: true + schema_pattern: tenant_* + exclude_schemas: + - tenant_c + tables: + - events +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + m := msgMeta{} + m.dbSchema, _ = msg.MetaGet("database_schema") + m.table, _ = msg.MetaGet("table") + m.operation, _ = msg.MetaGet("operation") + m.lsn, _ = msg.MetaGet("lsn") + collected = append(collected, m) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + // Wait for the 3 snapshot rows from the two non-excluded schemas; tenant_c's + // row must never contribute to this count. + assert.Eventually(t, func() bool { + return collectedLen() >= 3 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for snapshot rows") + + // Insert CDC rows into all three schemas, including the excluded one. + _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('dave')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_b.events (name) VALUES ('eve')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_c.events (name) VALUES ('trudy')") + require.NoError(t, err) + + // Wait for the 2 CDC rows from the non-excluded schemas (total 5). + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.Equal(c, 5, collectedLen()) + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for CDC rows") + + // tenant_c's CDC insert above raced the same replication stream as the + // tenant_a/tenant_b inserts already confirmed above, so if it were going + // to leak through it would have by now; assert the count never climbs + // past 5 to catch a delayed leak instead of just checking once. + assert.Never(t, func() bool { + return collectedLen() > 5 + }, 3*time.Second, 200*time.Millisecond, "received unexpected message(s) from excluded schema tenant_c") + + mu.Lock() + defer mu.Unlock() + + require.Len(t, collected, 5) + for _, m := range collected { + assert.NotEqual(t, "tenant_c", m.dbSchema, "tenant_c is excluded and must never appear, got message: %+v", m) + } + + var snapshots, cdcMsgs []msgMeta + for _, m := range collected { + if m.operation == "read" { + snapshots = append(snapshots, m) + } else { + cdcMsgs = append(cdcMsgs, m) + } + } + + // Snapshot assertions. + require.Len(t, snapshots, 3) + snapshotSchemas := make(map[string]int) + for _, m := range snapshots { + assert.Equal(t, "events", m.table, "snapshot: table should be bare name without schema prefix") + assert.Empty(t, m.lsn, "snapshot rows have no LSN") + snapshotSchemas[m.dbSchema]++ + } + assert.Equal(t, 2, snapshotSchemas["tenant_a"], "expected 2 snapshot rows from tenant_a") + assert.Equal(t, 1, snapshotSchemas["tenant_b"], "expected 1 snapshot row from tenant_b") + + // CDC assertions. + require.Len(t, cdcMsgs, 2) + cdcSchemas := make(map[string]int) + for _, m := range cdcMsgs { + assert.Equal(t, "insert", m.operation) + assert.Equal(t, "events", m.table) + assert.NotEmpty(t, m.lsn, "CDC rows must have an LSN") + cdcSchemas[m.dbSchema]++ + } + assert.Equal(t, 1, cdcSchemas["tenant_a"], "expected 1 CDC row from tenant_a") + assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") +} + func TestIntegrationMultiSchemaMissingTableDegradesGracefully(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") diff --git a/internal/impl/postgresql/pglogicalstream/config.go b/internal/impl/postgresql/pglogicalstream/config.go index f6cb5ce99c..2247421542 100644 --- a/internal/impl/postgresql/pglogicalstream/config.go +++ b/internal/impl/postgresql/pglogicalstream/config.go @@ -30,7 +30,11 @@ type Config struct { // non-empty, it takes precedence over DBSchema and schemas are resolved // dynamically at stream creation time. DBSchemaPattern string - DBTables []string + // DBExcludeSchemas is a list of schema names or glob patterns (same syntax + // as DBSchemaPattern) excluded from the schemas resolved by + // DBSchemaPattern. Only meaningful when DBSchemaPattern is non-empty. + DBExcludeSchemas []string + DBTables []string // Refreshes short lived IAM auth token that is treated as a password RefreshAuthToken func(ctx context.Context) error // ReplicationSlotName is the name of the replication slot to use diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 669acbc66a..a5fe4369b3 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -104,6 +104,36 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { if err != nil { return nil, fmt.Errorf("resolving schema pattern %q: %w", config.DBSchemaPattern, err) } + + if len(config.DBExcludeSchemas) > 0 { + // Filtering happens entirely against the schemas slice we already + // fetched above - no extra DB round-trips per exclude pattern. + var excluded []string + remaining := make([]string, 0, len(schemas)) + for _, schema := range schemas { + var isExcluded bool + for _, pattern := range config.DBExcludeSchemas { + matched, err := schemaMatchesExcludePattern(schema, pattern) + if err != nil { + return nil, fmt.Errorf("evaluating exclude_schemas pattern %q against schema %q: %w", pattern, schema, err) + } + if matched { + isExcluded = true + break + } + } + if isExcluded { + excluded = append(excluded, schema) + continue + } + remaining = append(remaining, schema) + } + if len(excluded) > 0 { + config.Logger.Infof("exclude_schemas %v excluded %d schema(s) %v from schema pattern %q; %d schema(s) remain: %v", config.DBExcludeSchemas, len(excluded), excluded, config.DBSchemaPattern, len(remaining), remaining) + } + schemas = remaining + } + if len(inaccessibleSchemas) > 0 && len(config.DBTables) > 0 { config.Logger.Warnf("schema pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaPattern, inaccessibleSchemas) } diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 4737f7c4cb..f8d6991007 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -11,6 +11,7 @@ package pglogicalstream import ( "context" "fmt" + "regexp" "strings" "github.com/jackc/pgx/v5/pgconn" @@ -198,3 +199,53 @@ func escapeLike(s string) string { } return b.String() } + +// schemaMatchesExcludePattern reports whether quotedSchemaName - a schema +// identifier in the same quoted form resolveSchemas returns - matches +// excludePattern, an exclude_schemas entry written in the same shape as a +// schema_pattern value (an exact name, a '*' glob, or a double-quoted exact +// identifier). +// +// Matching happens entirely in memory against a schema list we've already +// resolved, unlike resolveSchemas which queries the database: exclude_schemas +// only ever narrows a candidate set that's already been fetched, so there's +// no reason to pay for another round-trip per exclude pattern. Semantics +// mirror schemaPatternToLike without going through SQL: a quoted pattern is +// an exact, case-sensitive match on the unquoted name; an unquoted pattern is +// matched case-insensitively with '*' as a wildcard. +// +// Returns an error only when a quoted operand fails to unquote. This should +// only happen for a malformed excludePattern in practice, since +// quotedSchemaName is always freshly quoted by resolveSchemas. +func schemaMatchesExcludePattern(quotedSchemaName, excludePattern string) (bool, error) { + schemaName, err := sanitize.UnquotePostgresIdentifier(quotedSchemaName) + if err != nil { + return false, fmt.Errorf("unquoting schema identifier %q: %w", quotedSchemaName, err) + } + + if strings.HasPrefix(excludePattern, `"`) { + unquoted, err := sanitize.UnquotePostgresIdentifier(excludePattern) + if err != nil { + return false, fmt.Errorf("invalid quoted schema identifier %q: %w", excludePattern, err) + } + return schemaName == unquoted, nil + } + + re, err := globToRegexp(strings.ToLower(excludePattern)) + if err != nil { + return false, fmt.Errorf("invalid exclude pattern %q: %w", excludePattern, err) + } + return re.MatchString(strings.ToLower(schemaName)), nil +} + +// globToRegexp compiles an unquoted glob pattern (using '*' as a wildcard) +// into an anchored regexp - the in-memory equivalent of globToLike for +// callers matching against values already held in Go rather than via a SQL +// LIKE clause. +func globToRegexp(pattern string) (*regexp.Regexp, error) { + parts := strings.Split(pattern, "*") + for i, part := range parts { + parts[i] = regexp.QuoteMeta(part) + } + return regexp.Compile("^" + strings.Join(parts, ".*") + "$") +} diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go b/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go index 7ee213f6e4..7a352c6949 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go @@ -88,3 +88,43 @@ func TestEscapeLike(t *testing.T) { }) } } + +func TestSchemaMatchesExcludePattern(t *testing.T) { + tests := []struct { + name string + schema string + pattern string + expected bool + errContains string + }{ + // Unquoted patterns - case-insensitive, '*' as wildcard. + {name: "unquoted exact match", schema: `"tenant_a"`, pattern: "tenant_a", expected: true}, + {name: "unquoted exact no match", schema: `"tenant_a"`, pattern: "tenant_b", expected: false}, + {name: "unquoted glob match", schema: `"tenant_test_x"`, pattern: "tenant_test_*", expected: true}, + {name: "unquoted glob no match", schema: `"tenant_prod_x"`, pattern: "tenant_test_*", expected: false}, + {name: "bare wildcard matches everything", schema: `"anything"`, pattern: "*", expected: true}, + // Case-folding: unquoted patterns and unquoted-origin schema names both + // fold to lower-case, mirroring PostgreSQL's identifier folding. + {name: "case-insensitive exact match", schema: `"tenant_a"`, pattern: "TENANT_A", expected: true}, + {name: "case-insensitive glob match", schema: `"Tenant_Test_X"`, pattern: "tenant_test_*", expected: true}, + // Quoted patterns - exact, case-sensitive, no wildcard expansion. + {name: "quoted exact case-sensitive match", schema: `"MySchema"`, pattern: `"MySchema"`, expected: true}, + {name: "quoted exact case mismatch does not match", schema: `"MySchema"`, pattern: `"myschema"`, expected: false}, + {name: "quoted pattern does not expand wildcard", schema: `"tenant_a"`, pattern: `"tenant_*"`, expected: false}, + // Errors - only from a malformed pattern, never from the candidate + // schema name, since that's always freshly quoted by resolveSchemas. + {name: "unterminated quoted pattern errors", schema: `"tenant_a"`, pattern: `"unterminated`, errContains: "invalid quoted schema identifier"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := schemaMatchesExcludePattern(tt.schema, tt.pattern) + if tt.errContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errContains) + return + } + require.NoError(t, err) + assert.Equal(t, tt.expected, got) + }) + } +} diff --git a/internal/impl/postgresql/tests/current/Taskfile.yaml b/internal/impl/postgresql/tests/current/Taskfile.yaml index d81e5413ce..0b57a7d25d 100644 --- a/internal/impl/postgresql/tests/current/Taskfile.yaml +++ b/internal/impl/postgresql/tests/current/Taskfile.yaml @@ -47,10 +47,11 @@ tasks: # ── Test data ───────────────────────────────────────────────────────────────── data:insert: - desc: Insert CDC rows into both tenant schemas (triggers insert events) + desc: Insert CDC rows into all three tenant schemas (tenant_c must not appear in the output — it's excluded) cmds: - psql "{{.PG_DSN}}" -c "INSERT INTO tenant_a.events (name) VALUES ('dave'), ('eve');" - psql "{{.PG_DSN}}" -c "INSERT INTO tenant_b.events (name) VALUES ('frank');" + - psql "{{.PG_DSN}}" -c "INSERT INTO tenant_c.events (name) VALUES ('trudy');" data:update: desc: Update a row in tenant_a (triggers update event with 'before' field) @@ -89,6 +90,22 @@ tasks: # Note: an empty schema_pattern is NOT an error - it means "unset", and # the connector falls back to the schema field (default "public"). + test:exclude-schemas-requires-pattern: + desc: Confirm exclude_schemas is rejected at startup when schema_pattern is left unset + env: + PG_DSN: '{{.PG_DSN}}' + cmds: + - | + set +e + go run ../../../../../cmd/redpanda-connect/main.go run \ + ./test_config_exclude_no_pattern.yaml 2>&1 | head -5 + echo "exit $?" + # Expects: "exclude_schemas requires schema_pattern to be set" error + # printed, process exits non-zero. Uses a dedicated fixture file rather + # than --set on test_config.yaml, since --set rejects an empty RHS + # ("foo=" -> "expected foo=bar syntax"), so schema_pattern can't be + # cleared that way. + # ── Quick sanity ────────────────────────────────────────────────────────────── psql: diff --git a/internal/impl/postgresql/tests/current/setup.sql b/internal/impl/postgresql/tests/current/setup.sql index 512f8373fa..3c0c6264fd 100644 --- a/internal/impl/postgresql/tests/current/setup.sql +++ b/internal/impl/postgresql/tests/current/setup.sql @@ -1,10 +1,15 @@ -- Multi-schema CDC test setup --- Tests: schema glob (tenant_*), database_schema metadata, commit_ts_ms, before (update/delete) +-- Tests: schema glob (tenant_*), exclude_schemas (tenant_c), database_schema +-- metadata, commit_ts_ms, before (update/delete) -- ── Tenant schemas ──────────────────────────────────────────────────────────── +-- tenant_a and tenant_b are replicated; tenant_c matches the tenant_* glob in +-- test_config.yaml but is carved out via exclude_schemas — its rows must +-- never appear in the pipeline output. CREATE SCHEMA IF NOT EXISTS tenant_a; CREATE SCHEMA IF NOT EXISTS tenant_b; +CREATE SCHEMA IF NOT EXISTS tenant_c; -- ── Events table (same shape in each schema) ────────────────────────────────── @@ -22,12 +27,23 @@ CREATE TABLE IF NOT EXISTS tenant_b.events ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +CREATE TABLE IF NOT EXISTS tenant_c.events ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + -- REPLICA IDENTITY FULL so update/delete messages carry the full before-row. ALTER TABLE tenant_a.events REPLICA IDENTITY FULL; ALTER TABLE tenant_b.events REPLICA IDENTITY FULL; +ALTER TABLE tenant_c.events REPLICA IDENTITY FULL; -- ── Seed snapshot rows ──────────────────────────────────────────────────────── -- These are visible during the initial snapshot (stream_snapshot: true). +-- tenant_c's row (mallory) must NOT appear in the output — see exclude_schemas +-- in test_config.yaml. INSERT INTO tenant_a.events (name) VALUES ('alice'), ('bob'); INSERT INTO tenant_b.events (name) VALUES ('carol'); +INSERT INTO tenant_c.events (name) VALUES ('mallory'); diff --git a/internal/impl/postgresql/tests/current/test_config.yaml b/internal/impl/postgresql/tests/current/test_config.yaml index 838321a392..534b11496e 100644 --- a/internal/impl/postgresql/tests/current/test_config.yaml +++ b/internal/impl/postgresql/tests/current/test_config.yaml @@ -5,6 +5,10 @@ input: stream_snapshot: true # Glob pattern: replicates both tenant_a and tenant_b via one slot. schema_pattern: tenant_* + # tenant_c matches the glob above but is carved out here — its rows must + # never appear in the output below, neither during snapshot nor CDC. + exclude_schemas: + - tenant_c tables: - events diff --git a/internal/impl/postgresql/tests/current/test_config_exclude_no_pattern.yaml b/internal/impl/postgresql/tests/current/test_config_exclude_no_pattern.yaml new file mode 100644 index 0000000000..57c0caf983 --- /dev/null +++ b/internal/impl/postgresql/tests/current/test_config_exclude_no_pattern.yaml @@ -0,0 +1,22 @@ +# Negative-validation fixture for exclude_schemas: schema_pattern is +# deliberately left unset. --set can't assign an empty string (the CLI +# rejects "foo=" as "expected foo=bar syntax"), so this exists as its own +# file instead of overriding test_config.yaml at run time. +input: + postgres_cdc: + dsn: ${PG_DSN:postgres://postgres:postgres@localhost:5433/testdb?sslmode=disable} + slot_name: exclude_schemas_no_pattern_slot + exclude_schemas: + - tenant_c + tables: + - events + +output: + stdout: + codec: lines + +logger: + level: INFO + +metrics: + none: {} From 7a136ba4cdb655f6fc339550801e24323f883e99 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 12 Aug 2026 13:07:22 +0100 Subject: [PATCH 28/48] postgres_cdc: fix failing signalling tests --- internal/impl/postgresql/signaller_integration_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/impl/postgresql/signaller_integration_test.go b/internal/impl/postgresql/signaller_integration_test.go index fb70c7ff2d..d3ef8a5a2f 100644 --- a/internal/impl/postgresql/signaller_integration_test.go +++ b/internal/impl/postgresql/signaller_integration_test.go @@ -449,6 +449,7 @@ func startSignallingStream(t *testing.T, inputYAML string) (*pgtest.ReceivedMess } delete(m, "schema") delete(m, "commit_ts_ms") + delete(m, "database_schema") received.Add(m) } return nil From 52dbd2bb8984ec4aff9a3a769cdc05026ec7e9c7 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 12 Aug 2026 13:18:01 +0100 Subject: [PATCH 29/48] postgres_cdc: add further test coverage for uuid schemas --- .../pglogicalstream/schema_resolver.go | 47 ++++++---- .../schema_resolver_integration_test.go | 90 +++++++++++++++++++ .../pglogicalstream/schema_resolver_test.go | 34 +++---- 3 files changed, 140 insertions(+), 31 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index f8d6991007..da3958a73f 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -20,20 +20,28 @@ import ( ) // schemaPatternToLike converts a schema name or glob pattern into the LIKE -// pattern used by resolveSchemas. Extracted for unit testing. +// pattern used by resolveSchemas, plus whether that pattern must be matched +// case-sensitively. Extracted for unit testing. // -// For quoted identifiers the inner name is exact-escaped (no wildcard expansion). -// For unquoted patterns the '*' wildcard is converted to '%' and the input is -// folded to lower-case to match PostgreSQL's identifier folding. -func schemaPatternToLike(pattern string) (string, error) { +// For quoted identifiers the inner name is exact-escaped (no wildcard +// expansion) and matched case-sensitively, since a quoted identifier's case +// is significant and PostgreSQL does not fold it. For unquoted patterns the +// '*' wildcard is converted to '%' and matching is case-insensitive: this +// mirrors PostgreSQL folding unquoted identifiers to lower-case at creation +// time for the common case, but must hold even when a schema had to be +// created with a quoted identifier for an unrelated reason (e.g. a +// UUID-suffixed tenant schema, which requires quoting because hyphens are +// invalid in unquoted identifiers) and so kept whatever case it was written +// with — an unquoted glob like "tenant_*" is still expected to match it. +func schemaPatternToLike(pattern string) (likePattern string, caseSensitive bool, err error) { if strings.HasPrefix(pattern, `"`) { unquoted, err := sanitize.UnquotePostgresIdentifier(pattern) if err != nil { - return "", fmt.Errorf("invalid quoted schema identifier %q: %w", pattern, err) + return "", false, fmt.Errorf("invalid quoted schema identifier %q: %w", pattern, err) } - return escapeLike(unquoted), nil + return escapeLike(unquoted), true, nil } - return globToLike(strings.ToLower(pattern)), nil + return globToLike(strings.ToLower(pattern)), false, nil } // resolveSchemas returns the schemas matching pattern that the connection's @@ -45,23 +53,30 @@ func schemaPatternToLike(pattern string) (string, error) { // see yet. // // For unquoted patterns (e.g. "tenant_*") the pattern is matched -// case-insensitively via LIKE, because PostgreSQL folds unquoted identifiers -// to lower-case at creation time. For quoted identifiers (e.g. `"MySchema"`) -// an exact case-sensitive lookup is performed. System schemas (pg_* and -// information_schema) are always excluded so that wildcard patterns like "*" -// do not attempt to replicate catalog tables. +// case-insensitively via ILIKE, regardless of whether the matched schema was +// itself created case-insensitively. For quoted identifiers (e.g. +// `"MySchema"`) an exact case-sensitive lookup is performed via LIKE. System +// schemas (pg_* and information_schema) are always excluded case-sensitively +// so that wildcard patterns like "*" do not attempt to replicate catalog +// tables. // // Returned schema names are quoted PostgreSQL identifiers. Returns an error // if either query fails; returns a nil visibleSchemas slice (with a nil err) // if no schemas match — callers should treat that as an error condition. func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (visibleSchemas, inaccessibleSchemas []string, err error) { - likePattern, err := schemaPatternToLike(pattern) + likePattern, caseSensitive, err := schemaPatternToLike(pattern) if err != nil { return nil, nil, err } + // Fixed, code-chosen operator (never derived from user input), so it's + // safe to splice directly into the query text rather than parameterize. + op := "ILIKE" + if caseSensitive { + op = "LIKE" + } q, err := sanitize.SQLQuery( - "SELECT schema_name FROM information_schema.schemata WHERE schema_name LIKE $1 ESCAPE '!' AND schema_name NOT LIKE 'pg!_%' ESCAPE '!' AND schema_name != 'information_schema'", + fmt.Sprintf("SELECT schema_name FROM information_schema.schemata WHERE schema_name %s $1 ESCAPE '!' AND schema_name NOT LIKE 'pg!_%%' ESCAPE '!' AND schema_name != 'information_schema'", op), likePattern, ) if err != nil { @@ -90,7 +105,7 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (v // missing from information_schema.schemata means the role lacks USAGE (or // similar) on that schema rather than the schema simply not existing. nsQ, err := sanitize.SQLQuery( - "SELECT nspname FROM pg_catalog.pg_namespace WHERE nspname LIKE $1 ESCAPE '!' AND nspname NOT LIKE 'pg!_%' ESCAPE '!' AND nspname != 'information_schema'", + fmt.Sprintf("SELECT nspname FROM pg_catalog.pg_namespace WHERE nspname %s $1 ESCAPE '!' AND nspname NOT LIKE 'pg!_%%' ESCAPE '!' AND nspname != 'information_schema'", op), likePattern, ) if err != nil { diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go b/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go index 6af1d4ad7e..91243a6802 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go @@ -69,3 +69,93 @@ func TestIntegrationResolveSchemasReportsInaccessibleSchemas(t *testing.T) { assert.Equal(t, []string{`"visible_schema"`}, visible) assert.Equal(t, []string{`"hidden_schema"`}, inaccessible) } + +func TestIntegrationResolveSchemasUUIDSuffixedSchemas(t *testing.T) { + integration.CheckSkip(t) + + _, adminURL := createDockerInstance(t) + + adminDB, err := sql.Open("postgres", adminURL) + require.NoError(t, err) + defer adminDB.Close() + + // Quoting is mandatory here because of the UUID's hyphens, regardless of + // case - so both of these preserve their literal casing exactly as written. + const ( + lowerCaseSchema = `"tenant_a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"` + mixedCaseSchema = `"Tenant_9c0b4ef8-bb6d-6bb9-bd38-0a11a0eebc99"` + ) + _, err = adminDB.Exec(`CREATE SCHEMA ` + lowerCaseSchema) + require.NoError(t, err) + _, err = adminDB.Exec(`CREATE SCHEMA ` + mixedCaseSchema) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + conn, err := pgconn.Connect(ctx, adminURL) + require.NoError(t, err) + defer closeConn(t, conn) + + visible, inaccessible, err := resolveSchemas(ctx, conn, "tenant_*") + require.NoError(t, err) + + // Both schemas match the unquoted glob despite the case difference in + // their literal prefix - matching is case-insensitive independent of how + // the schema itself was created. + assert.ElementsMatch(t, []string{lowerCaseSchema, mixedCaseSchema}, visible) + assert.Empty(t, inaccessible) + + // A quoted pattern is still exact and case-sensitive: it picks out only + // the schema whose case matches the pattern, with no wildcard expansion. + exactVisible, _, err := resolveSchemas(ctx, conn, mixedCaseSchema) + require.NoError(t, err) + assert.Equal(t, []string{mixedCaseSchema}, exactVisible) +} + +func TestIntegrationResolveSchemasBareUUIDSchema(t *testing.T) { + integration.CheckSkip(t) + + _, adminURL := createDockerInstance(t) + + adminDB, err := sql.Open("postgres", adminURL) + require.NoError(t, err) + defer adminDB.Close() + + // Upper-case hex digits, no prefix - quoting is mandatory purely because + // of the hyphens, not because of anything alphabetic. + const bareUUIDSchema = `"A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11"` + _, err = adminDB.Exec(`CREATE SCHEMA ` + bareUUIDSchema) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + conn, err := pgconn.Connect(ctx, adminURL) + require.NoError(t, err) + defer closeConn(t, conn) + + // A bare wildcard has no literal characters to case-compare, so this is + // unaffected by hex-digit casing either way - included as a baseline. + visible, _, err := resolveSchemas(ctx, conn, "*") + require.NoError(t, err) + assert.Contains(t, visible, bareUUIDSchema) + + // The interesting case: an unquoted pattern whose only literal portion is + // a lower-case chunk of the UUID itself (no prefix) must still match the + // upper-case schema case-insensitively. + visible, _, err = resolveSchemas(ctx, conn, "a0eebc99-*") + require.NoError(t, err) + assert.Equal(t, []string{bareUUIDSchema}, visible) + + // Same, but the literal chunk sits in the middle rather than at the start. + visible, _, err = resolveSchemas(ctx, conn, "*-bb6d-*") + require.NoError(t, err) + assert.Equal(t, []string{bareUUIDSchema}, visible) + + // A quoted pattern remains an exact, case-sensitive lookup: the + // differently-cased quoted form matches nothing. + visible, _, err = resolveSchemas(ctx, conn, `"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"`) + require.NoError(t, err) + assert.Empty(t, visible) +} diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go b/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go index 7a352c6949..ae1ee89a92 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go @@ -38,27 +38,30 @@ func TestGlobToLike(t *testing.T) { func TestSchemaPatternToLike(t *testing.T) { tests := []struct { - pattern string - expected string - errContains string + pattern string + expected string + caseSensitive bool + errContains string }{ - // Unquoted glob patterns — folded to lower-case, '*' → '%', '_' escaped. - {"public", "public", ""}, - {"tenant_*", "tenant!_%", ""}, - {"*", "%", ""}, - {"schema_1", "schema!_1", ""}, + // Unquoted glob patterns — folded to lower-case, '*' → '%', '_' escaped, + // matched case-insensitively regardless of how the matched schema was created. + {pattern: "public", expected: "public"}, + {pattern: "tenant_*", expected: "tenant!_%"}, + {pattern: "*", expected: "%"}, + {pattern: "schema_1", expected: "schema!_1"}, // Upper-case is folded: TENANT_* matches the same rows as tenant_*. - {"TENANT_*", "tenant!_%", ""}, - // Quoted exact identifier — case preserved, no wildcard expansion. - {`"MySchema"`, "MySchema", ""}, - {`"schema_1"`, "schema!_1", ""}, - {`"has%bang!"`, "has!%bang!!", ""}, + {pattern: "TENANT_*", expected: "tenant!_%"}, + // Quoted exact identifier — case preserved, no wildcard expansion, + // matched case-sensitively. + {pattern: `"MySchema"`, expected: "MySchema", caseSensitive: true}, + {pattern: `"schema_1"`, expected: "schema!_1", caseSensitive: true}, + {pattern: `"has%bang!"`, expected: "has!%bang!!", caseSensitive: true}, // Unterminated quoted identifier → error. - {`"bad`, "", "invalid quoted schema identifier"}, + {pattern: `"bad`, errContains: "invalid quoted schema identifier"}, } for _, tt := range tests { t.Run(tt.pattern, func(t *testing.T) { - got, err := schemaPatternToLike(tt.pattern) + got, caseSensitive, err := schemaPatternToLike(tt.pattern) if tt.errContains != "" { require.Error(t, err) assert.Contains(t, err.Error(), tt.errContains) @@ -66,6 +69,7 @@ func TestSchemaPatternToLike(t *testing.T) { } require.NoError(t, err) assert.Equal(t, tt.expected, got) + assert.Equal(t, tt.caseSensitive, caseSensitive) }) } } From 55cf17dda5c835ff935ddeb77ee4f5c13c059f7f Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 12 Aug 2026 16:21:42 +0100 Subject: [PATCH 30/48] postgres_cdc: adress qupted schema --- .../components/pages/inputs/postgres_cdc.adoc | 4 +- internal/impl/postgresql/input_pg_stream.go | 20 +++-- .../impl/postgresql/input_pg_stream_test.go | 10 ++- internal/impl/postgresql/integration_test.go | 85 +++++++++++++++++++ 4 files changed, 109 insertions(+), 10 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 7946e3e547..67dd5f4dd0 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -202,6 +202,8 @@ Schema pattern matching runs once at pipeline startup. Schemas created after the If `tables` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when `tables` is left empty - see `tables` below. +This pattern is matched against schema names as stored in the database rather than parsed as an identifier, so unquoted patterns may contain characters that are not valid in an unquoted PostgreSQL identifier (e.g. `tenant-*` or `a0eebc99-*` to match a UUID-suffixed schema that had to be created with a quoted identifier). + This field is mutually exclusive with `schema`; when set, it takes over schema resolution entirely and `schema` must be left at its default. @@ -623,7 +625,7 @@ Optional external ID for the role assumption. === `signal_table_name` -The name of the table used to send control signals to the connector, excluding the schema. The table must +The name of the table used to send control signals to the connector, excluding the schema. Not supported when `schema_pattern` is set, since there is no single schema to anchor the signal table to. The table must exist in the schema configured via the `schema` field, and must not also appear in `tables` — the signal table is implicitly added to the publication and excluded from snapshot scans, so listing it in both places is rejected at startup. It must have at least these columns — startup validation checks diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 4eba744b76..d19f659835 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -137,6 +137,8 @@ Schema pattern matching runs once at pipeline startup. Schemas created after the If `+"`"+fieldTables+"`"+` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when `+"`"+fieldTables+"`"+` is left empty - see `+"`"+fieldTables+"`"+` below. +This pattern can contain characters that wouldn't be allowed in an unquoted schema name, because it's only ever compared against the real name of each schema in the database - it doesn't have to be a valid name itself. For example, a schema literally named `+"`a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11`"+` (which must have been created using double quotes, since hyphens aren't allowed in an unquoted `+"`CREATE SCHEMA`"+` statement) can still be matched using the unquoted pattern `+"`a0eebc99-*`"+`. + This field is mutually exclusive with `+"`"+fieldSchema+"`"+`; when set, it takes over schema resolution entirely and `+"`"+fieldSchema+"`"+` must be left at its default.`). Examples("tenant_*", "*", `"MyCaseSensitiveSchemaNeedingQuotes"`). Optional(). @@ -535,8 +537,15 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser } // validateSchemaPattern validates a schema name or glob pattern. -// Accepts exact postgres identifiers (letters/digits/underscores) and glob -// patterns that additionally allow '*' as a wildcard character. +// +// Unquoted patterns are matched via ILIKE against stored schema names (see +// resolveSchemas) rather than parsed as an identifier we construct +// ourselves, so the accepted character set here is deliberately as wide as +// what's legal inside a quoted schema name, plus '*' as a wildcard - it is +// not narrowed to unquoted-identifier syntax. This matters because a schema +// that had to be created with a quoted identifier (e.g. a UUID-suffixed +// tenant schema, since hyphens are invalid in unquoted identifiers) must +// still be matchable via an unquoted glob such as "tenant-*" or "a0eebc99-*". // Double-quoted identifiers (e.g. "MySchema") are accepted as exact names; // wildcards are not allowed inside quotes. func validateSchemaPattern(s string) error { @@ -552,11 +561,8 @@ func validateSchemaPattern(s string) error { } return nil } - for i, ch := range s { - if unicode.IsLetter(ch) || unicode.IsDigit(ch) || ch == '_' || ch == '*' { - continue - } - return fmt.Errorf("invalid character %q at position %d in schema pattern %q", ch, i, s) + if strings.ContainsRune(s, '"') { + return fmt.Errorf("unquoted schema pattern %q must not contain '\"'", s) } first, _ := utf8.DecodeRuneInString(s) if first != '_' && first != '*' && !unicode.IsLetter(first) { diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index 53aad6f05c..73a3874122 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 Redpanda Data, Inc. +// Copyright 2026 Redpanda Data, Inc. // // Licensed as a Redpanda Enterprise file under the Redpanda Community // License (the "License"); you may not use this file except in compliance with @@ -67,8 +67,14 @@ func TestSchemaPatternValidation(t *testing.T) { {`""`, "invalid quoted schema identifier"}, {"1abc", "must start with a letter"}, {`"unclosed`, "invalid quoted schema identifier"}, - {"schema-name", "invalid character"}, + // Regression test: an unquoted pattern is matched against stored + // schema names, not parsed as an identifier, so hyphens (invalid in + // unquoted Postgres identifiers) must still be accepted - e.g. to + // match a UUID-suffixed tenant schema that had to be created quoted. + {"schema-name", ""}, + {"a0eebc99-*", ""}, {`"quoted*"`, "wildcard"}, + {`a"b`, `must not contain '"'`}, } for _, tt := range tests { t.Run(tt.pattern, func(t *testing.T) { diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 6f7f802359..a795c54b5e 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1898,6 +1898,91 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } +func TestIntegrationSchemaPatternMatchesHyphenatedUUIDSchema(t *testing.T) { + integration.CheckSkip(t) + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + const uuidSchema = "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11" + _, err = db.Exec(fmt.Sprintf(`CREATE SCHEMA "%s"`, uuidSchema)) + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf(`CREATE TABLE "%s".events (id SERIAL PRIMARY KEY, name TEXT)`, uuidSchema)) + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf(`INSERT INTO "%s".events (name) VALUES ('alice')`, uuidSchema)) + require.NoError(t, err) + + type msgMeta struct { + dbSchema string + table string + operation string + } + + var ( + mu sync.Mutex + collected []msgMeta + ) + collectedLen := func() int { + mu.Lock() + defer mu.Unlock() + return len(collected) + } + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: hyphenated_schema_pattern_slot + stream_snapshot: true + schema_pattern: a0eebc99-* + tables: + - events +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + m := msgMeta{} + m.dbSchema, _ = msg.MetaGet("database_schema") + m.table, _ = msg.MetaGet("table") + m.operation, _ = msg.MetaGet("operation") + collected = append(collected, m) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + assert.Eventually(t, func() bool { + return collectedLen() >= 1 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for snapshot row from hyphenated UUID schema") + + _, err = db.Exec(fmt.Sprintf(`INSERT INTO "%s".events (name) VALUES ('bob')`, uuidSchema)) + require.NoError(t, err) + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.Equal(c, 2, collectedLen()) + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for CDC row from hyphenated UUID schema") + + mu.Lock() + defer mu.Unlock() + require.Len(t, collected, 2) + for _, m := range collected { + assert.Equal(t, uuidSchema, m.dbSchema, "database_schema metadata should be the raw, unquoted, case-preserved schema name") + assert.Equal(t, "events", m.table) + } +} + func TestIntegrationMultiSchemaMissingTableDegradesGracefully(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") From f39ff341595d724c78ae5d052eb8ee0ab631801b Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 12 Aug 2026 16:42:48 +0100 Subject: [PATCH 31/48] postgres_cdc: allow a leading digit for unquoted patterns --- internal/impl/postgresql/input_pg_stream.go | 17 ++++------------- .../impl/postgresql/input_pg_stream_test.go | 14 +++++++++++++- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index d19f659835..05fdf13420 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -18,8 +18,6 @@ import ( "strings" "sync" "time" - "unicode" - "unicode/utf8" "github.com/Jeffail/checkpoint" "github.com/Jeffail/shutdown" @@ -539,13 +537,10 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser // validateSchemaPattern validates a schema name or glob pattern. // // Unquoted patterns are matched via ILIKE against stored schema names (see -// resolveSchemas) rather than parsed as an identifier we construct -// ourselves, so the accepted character set here is deliberately as wide as -// what's legal inside a quoted schema name, plus '*' as a wildcard - it is -// not narrowed to unquoted-identifier syntax. This matters because a schema -// that had to be created with a quoted identifier (e.g. a UUID-suffixed -// tenant schema, since hyphens are invalid in unquoted identifiers) must -// still be matchable via an unquoted glob such as "tenant-*" or "a0eebc99-*". +// resolveSchemas), not parsed as an identifier, so any character or leading +// character is accepted - including hyphens and leading digits - except a +// literal '"'. This lets a glob like "a0eebc99-*" match a UUID-suffixed +// schema that itself had to be created quoted. // Double-quoted identifiers (e.g. "MySchema") are accepted as exact names; // wildcards are not allowed inside quotes. func validateSchemaPattern(s string) error { @@ -564,10 +559,6 @@ func validateSchemaPattern(s string) error { if strings.ContainsRune(s, '"') { return fmt.Errorf("unquoted schema pattern %q must not contain '\"'", s) } - first, _ := utf8.DecodeRuneInString(s) - if first != '_' && first != '*' && !unicode.IsLetter(first) { - return fmt.Errorf("schema pattern %q must start with a letter, underscore, or '*'", s) - } return nil } diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index 73a3874122..bce903c4b7 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -65,7 +65,12 @@ func TestSchemaPatternValidation(t *testing.T) { // Regression test: len("") == 2 used to pass the old `len(s) < 2` guard. // Fixed to `len(s) < 3`. {`""`, "invalid quoted schema identifier"}, - {"1abc", "must start with a letter"}, + // Regression test: a leading digit is not an identifier-syntax + // violation here - the pattern is compared via ILIKE, never spliced + // into an identifier position - so "1abc" must be as valid as any + // other unquoted pattern. See the "9c0b4ef8-*" case below for the + // motivating real-world scenario (a UUID-suffixed tenant schema). + {"1abc", ""}, {`"unclosed`, "invalid quoted schema identifier"}, // Regression test: an unquoted pattern is matched against stored // schema names, not parsed as an identifier, so hyphens (invalid in @@ -73,6 +78,13 @@ func TestSchemaPatternValidation(t *testing.T) { // match a UUID-suffixed tenant schema that had to be created quoted. {"schema-name", ""}, {"a0eebc99-*", ""}, + // Regression test: most UUIDs begin with a hex digit, so a tenant + // schema named e.g. "9c0b4ef8-bb6d-6bb9-bd38-0a11a0eebc99" (created + // quoted, per the a0eebc99-* case above) must be matchable by an + // unquoted glob starting with a digit - the pattern is compared via + // ILIKE, never spliced into an identifier position, so there's no + // syntactic reason to require a letter/underscore/'*' lead-in. + {"9c0b4ef8-*", ""}, {`"quoted*"`, "wildcard"}, {`a"b`, `must not contain '"'`}, } From bbf65cdb27b60c0bbd59e1c529daaa2dfe6de538 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 12 Aug 2026 16:47:05 +0100 Subject: [PATCH 32/48] postgres_cdc: remove now redundant subtests --- internal/impl/postgresql/input_pg_stream_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index bce903c4b7..fe682e4cb9 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -155,9 +155,9 @@ func TestExcludeSchemasValidation(t *testing.T) { {"tenant_test", ""}, {"tenant_test_*", ""}, {`"MySchema"`, ""}, - {"1abc", "must start with a letter"}, + {"1abc", ""}, {`"unclosed`, "invalid quoted schema identifier"}, - {"schema-name", "invalid character"}, + {"schema-name", ""}, {`"quoted*"`, "wildcard"}, } for _, tt := range tests { From 715e23336d3ba2fe8c2997fb5a678e9c9bfdd2b5 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 12 Aug 2026 16:58:32 +0100 Subject: [PATCH 33/48] postgres_cdc: ensure we don't discover views --- internal/impl/postgresql/integration_test.go | 86 +++++++++++++++++++ .../pglogicalstream/schema_resolver.go | 21 +++-- 2 files changed, 98 insertions(+), 9 deletions(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index a795c54b5e..b91c99205c 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -2062,6 +2062,92 @@ postgres_cdc: assert.Equal(t, "events", collected[0].table) } +// TestIntegrationMultiSchemaViewNamesakeDegradesGracefully guards against a +// relation that exists but isn't publishable: tenant_c matches the schema +// glob and has an "events" view (e.g. a compatibility shim over a renamed +// table), not the "events" table configured. Before restricting +// resolveExistingTables to table_type = 'BASE TABLE', this view counted as +// present, so CreatePublication's FOR TABLE clause referenced it and failed +// setup for every matched schema with "... is not supported for views". +// tenant_c's view should instead be skipped with a warning, the same way a +// genuinely missing table is, leaving tenant_a free to stream. +func TestIntegrationMultiSchemaViewNamesakeDegradesGracefully(t *testing.T) { + integration.CheckSkip(t) + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + _, err = db.Exec("CREATE SCHEMA tenant_a") + require.NoError(t, err) + _, err = db.Exec("CREATE TABLE tenant_a.events (id SERIAL PRIMARY KEY, name TEXT)") + require.NoError(t, err) + _, err = db.Exec("CREATE SCHEMA tenant_c") + require.NoError(t, err) + _, err = db.Exec("CREATE VIEW tenant_c.events AS SELECT 1 AS id, 'namesake'::text AS name") + require.NoError(t, err) + + _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('alice')") + require.NoError(t, err) + + type msgMeta struct { + dbSchema string + table string + } + + var ( + mu sync.Mutex + collected []msgMeta + ) + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: view_namesake_degrade_slot + stream_snapshot: true + schema_pattern: tenant_* + tables: + - events +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + m := msgMeta{} + m.dbSchema, _ = msg.MetaGet("database_schema") + m.table, _ = msg.MetaGet("table") + collected = append(collected, m) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + // tenant_a should keep streaming even though tenant_c's "events" is a + // view rather than a publishable table. + assert.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(collected) >= 1 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for tenant_a snapshot row; a view namesake in tenant_c should not block replication or fail publication setup") + + mu.Lock() + defer mu.Unlock() + require.Len(t, collected, 1) + assert.Equal(t, "tenant_a", collected[0].dbSchema) + assert.Equal(t, "events", collected[0].table) +} + func TestIntegrationNoSchemasMatchedReturnsError(t *testing.T) { integration.CheckSkip(t) databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index da3958a73f..668314767e 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -130,15 +130,18 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (v return schemas, hidden, nil } -// resolveExistingTables returns the set of quoted table identifiers that -// actually exist in the given (already quoted) schema. +// resolveExistingTables returns the quoted names of the publishable base +// tables (including partitioned tables) that exist in the given (already +// quoted) schema. // -// This is used to resolve a schema glob × table list combination per-schema -// rather than assuming every matched schema contains every listed table. A -// schema matching the glob but missing one of the configured tables (e.g. a -// tenant schema that's still being provisioned) would otherwise cause -// CreatePublication's FOR TABLE clause to reference a non-existent relation, -// failing publication setup for every schema, not just the drifted one. +// Used to resolve a schema glob × table list combination per-schema, since a +// matched schema may not contain a configured table (e.g. still being +// provisioned) or may have a same-named view/foreign/temporary table +// instead. table_type is restricted to 'BASE TABLE' so either case is +// treated as missing and falls through to the caller's "not found, skipping" +// warning - otherwise CreatePublication's FOR TABLE clause would reference +// an unpublishable relation and fail setup for every matched schema, not +// just the drifted one. func resolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchema string) (map[string]struct{}, error) { schema, err := sanitize.UnquotePostgresIdentifier(quotedSchema) if err != nil { @@ -146,7 +149,7 @@ func resolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchem } q, err := sanitize.SQLQuery( - "SELECT table_name FROM information_schema.tables WHERE table_schema = $1", + "SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type = 'BASE TABLE'", schema, ) if err != nil { From 65b08a9d0b2a349688ac8a1724ab4e5ec8259e88 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 12 Aug 2026 17:08:13 +0100 Subject: [PATCH 34/48] postgres_cdc: require/assert switch --- internal/impl/postgresql/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index b91c99205c..dab3a68eda 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -2203,7 +2203,7 @@ postgres_cdc: // (rather than once upfront) so a row lands after startup completes. writer := asyncroutine.NewPeriodic(100*time.Millisecond, func() { _, err := db.Exec("INSERT INTO flights (name, created_at) VALUES ('alice', now());") - require.NoError(t, err) + assert.NoError(t, err) }) writer.Start() t.Cleanup(writer.Stop) From 6079ef8fa7fe95ecaf54e80b2f2a781d809f5174 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 12 Aug 2026 17:14:38 +0100 Subject: [PATCH 35/48] postgres_cdc: update docs --- docs/modules/components/pages/inputs/postgres_cdc.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 67dd5f4dd0..f8c013b09d 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -202,7 +202,7 @@ Schema pattern matching runs once at pipeline startup. Schemas created after the If `tables` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when `tables` is left empty - see `tables` below. -This pattern is matched against schema names as stored in the database rather than parsed as an identifier, so unquoted patterns may contain characters that are not valid in an unquoted PostgreSQL identifier (e.g. `tenant-*` or `a0eebc99-*` to match a UUID-suffixed schema that had to be created with a quoted identifier). +This pattern can contain characters that wouldn't be allowed in an unquoted schema name, because it's only ever compared against the real name of each schema in the database - it doesn't have to be a valid name itself. For example, a schema literally named `a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11` (which must have been created using double quotes, since hyphens aren't allowed in an unquoted `CREATE SCHEMA` statement) can still be matched using the unquoted pattern `a0eebc99-*`. This field is mutually exclusive with `schema`; when set, it takes over schema resolution entirely and `schema` must be left at its default. From a74fdf7853790dab7cdfd26fa0bf25b0534547c8 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Thu, 13 Aug 2026 09:21:29 +0300 Subject: [PATCH 36/48] postgres_cdc: group tables/schema var declarations --- internal/impl/postgresql/pglogicalstream/logical_stream.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 55913cec6f..99439abd3f 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -98,8 +98,10 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { return nil, err } - var tables []TableFQN - var schema string + var ( + tables []TableFQN + schema string + ) if config.DBSchemaPattern != "" { if config.SignalTableName != "" { return nil, errors.New("signal_table_name is not supported when schema_pattern is set") From 24aa2cd59daff86deb00805d134cc8799a860956 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Thu, 13 Aug 2026 09:43:06 +0300 Subject: [PATCH 37/48] postgres_cdc: auto-discover tables for schema_pattern when tables is empty --- .../components/pages/inputs/postgres_cdc.adoc | 10 +- internal/impl/postgresql/input_pg_stream.go | 12 +- internal/impl/postgresql/integration_test.go | 180 +++++++++++++++--- .../pglogicalstream/logical_stream.go | 47 +++-- 4 files changed, 200 insertions(+), 49 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index f8c013b09d..1539b17d55 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -145,7 +145,7 @@ When set to true, empty messages with operation types BEGIN and COMMIT are gener === `stream_snapshot` -When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized. Note that this has no effect if `tables` is left empty, since the snapshot is only planned for tables listed there. +When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized. Note that this has no effect if `tables` is left empty and `schema_pattern` is NOT set, since in that case the snapshot is only planned for tables listed in `tables`. When `schema_pattern` IS set, leaving `tables` empty auto-discovers tables to snapshot instead - see `tables` below - and every discovered table must have a primary key. *Type*: `bool` @@ -200,7 +200,7 @@ Double-quoted identifiers are treated as exact names and do not support wildcard Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted. -If `tables` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when `tables` is left empty - see `tables` below. +If this pattern matches no schema in the database, startup fails with an error - this holds whether or not `tables` is set. See `tables` below for what happens when it's left empty. This pattern can contain characters that wouldn't be allowed in an unquoted schema name, because it's only ever compared against the real name of each schema in the database - it doesn't have to be a valid name itself. For example, a schema literally named `a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11` (which must have been created using double quotes, since hyphens aren't allowed in an unquoted `CREATE SCHEMA` statement) can still be matched using the unquoted pattern `a0eebc99-*`. @@ -229,6 +229,8 @@ Each entry uses the same syntax as `schema_pattern`: an exact schema name, a glo A schema that matches `schema_pattern` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by `schema_pattern` is silently ignored, so a typo here simply excludes nothing rather than failing startup. +This exclusion is applied before `tables` is resolved, so it also takes effect when `tables` is left empty and tables are auto-discovered. + *Type*: `array` @@ -248,7 +250,9 @@ A list of table names to include in the logical replication. Each table should b When `schema_pattern` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. -If left empty, the underlying PostgreSQL publication is created `FOR ALL TABLES`, which replicates every table in every schema of the database, ignoring `schema`. This also disables `stream_snapshot`, since the initial snapshot is only planned for tables listed here. +If left empty while `schema_pattern` is set, every base table in each matched (and un-excluded, see `exclude_schemas`) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. + +If left empty while `schema_pattern` is NOT set, the underlying PostgreSQL publication is instead created `FOR ALL TABLES`, which replicates every table in every schema of the database, ignoring `schema`. This also disables `stream_snapshot`, since the initial snapshot is only planned for tables listed here. *Type*: `array` diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 05fdf13420..9ffda49895 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -104,7 +104,7 @@ This input adds the following metadata fields to each message: ShortDescription("Emit empty BEGIN and COMMIT messages at the start and end of each transaction."). Default(false)). Field(service.NewBoolField(fieldStreamSnapshot). - Description("When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized. Note that this has no effect if `" + fieldTables + "` is left empty, since the snapshot is only planned for tables listed there."). + Description("When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized. Note that this has no effect if `" + fieldTables + "` is left empty and `" + fieldSchemaPattern + "` is NOT set, since in that case the snapshot is only planned for tables listed in `" + fieldTables + "`. When `" + fieldSchemaPattern + "` IS set, leaving `" + fieldTables + "` empty auto-discovers tables to snapshot instead - see `" + fieldTables + "` below - and every discovered table must have a primary key."). ShortDescription("Stream a snapshot of all existing data before streaming changes. Snapshot tables must have a primary key."). Example(true). Default(false)). @@ -133,7 +133,7 @@ Double-quoted identifiers are treated as exact names and do not support wildcard Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted. -If `+"`"+fieldTables+"`"+` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when `+"`"+fieldTables+"`"+` is left empty - see `+"`"+fieldTables+"`"+` below. +If this pattern matches no schema in the database, startup fails with an error - this holds whether or not `+"`"+fieldTables+"`"+` is set. See `+"`"+fieldTables+"`"+` below for what happens when it's left empty. This pattern can contain characters that wouldn't be allowed in an unquoted schema name, because it's only ever compared against the real name of each schema in the database - it doesn't have to be a valid name itself. For example, a schema literally named `+"`a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11`"+` (which must have been created using double quotes, since hyphens aren't allowed in an unquoted `+"`CREATE SCHEMA`"+` statement) can still be matched using the unquoted pattern `+"`a0eebc99-*`"+`. @@ -147,7 +147,9 @@ This field is mutually exclusive with `+"`"+fieldSchema+"`"+`; when set, it take Each entry uses the same syntax as ` + "`" + fieldSchemaPattern + "`" + `: an exact schema name, a glob pattern using ` + "`*`" + ` as a wildcard, or a double-quoted exact identifier for an exact, case-sensitive match. -A schema that matches ` + "`" + fieldSchemaPattern + "`" + ` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by ` + "`" + fieldSchemaPattern + "`" + ` is silently ignored, so a typo here simply excludes nothing rather than failing startup.`). +A schema that matches ` + "`" + fieldSchemaPattern + "`" + ` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by ` + "`" + fieldSchemaPattern + "`" + ` is silently ignored, so a typo here simply excludes nothing rather than failing startup. + +This exclusion is applied before ` + "`" + fieldTables + "`" + ` is resolved, so it also takes effect when ` + "`" + fieldTables + "`" + ` is left empty and tables are auto-discovered.`). Examples([]string{"tenant_internal", "tenant_test_*"}). Optional(). Default([]string{}), @@ -157,7 +159,9 @@ A schema that matches ` + "`" + fieldSchemaPattern + "`" + ` and also matches an When ` + "`schema_pattern`" + ` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. -If left empty, the underlying PostgreSQL publication is created ` + "`FOR ALL TABLES`" + `, which replicates every table in every schema of the database, ignoring ` + "`" + fieldSchema + "`" + `. This also disables ` + "`" + fieldStreamSnapshot + "`" + `, since the initial snapshot is only planned for tables listed here.`). +If left empty while ` + "`schema_pattern`" + ` is set, every base table in each matched (and un-excluded, see ` + "`" + fieldExcludeSchemas + "`" + `) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. + +If left empty while ` + "`schema_pattern`" + ` is NOT set, the underlying PostgreSQL publication is instead created ` + "`FOR ALL TABLES`" + `, which replicates every table in every schema of the database, ignoring ` + "`" + fieldSchema + "`" + `. This also disables ` + "`" + fieldStreamSnapshot + "`" + `, since the initial snapshot is only planned for tables listed here.`). Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`})). Field(service.NewIntField(fieldCheckpointLimit). Description("The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given LSN will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees."). diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index dab3a68eda..07bcf48958 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -2182,36 +2182,99 @@ tables: assert.Contains(t, err.Error(), "no schemas found matching pattern") } -// TestIntegrationForAllTablesIgnoresNonMatchingSchemaPattern guards the -// documented behaviour of FOR ALL TABLES mode: when `tables` is left empty, -// `schema` has no effect and must not block startup even if it matches no -// schema in the database. -func TestIntegrationForAllTablesIgnoresNonMatchingSchemaPattern(t *testing.T) { +// TestIntegrationSchemaPatternNonMatchingFailsEvenWithEmptyTables guards the +// fixed behaviour: schema_pattern always scopes replication, even when +// `tables` is left empty. A schema_pattern matching nothing in the database +// must fail startup rather than silently falling back to a database-wide +// FOR ALL TABLES publication (the old behaviour, which made exclude_schemas +// meaningless whenever tables was left unset). +func TestIntegrationSchemaPatternNonMatchingFailsEvenWithEmptyTables(t *testing.T) { integration.CheckSkip(t) - databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) tmpl := fmt.Sprintf(` -postgres_cdc: - dsn: %s - slot_name: for_all_tables_ignores_schema_slot - schema_pattern: nonexistent_schema_zzz_* +dsn: %s +slot_name: no_schema_match_empty_tables_slot +schema_pattern: nonexistent_schema_zzz_* `, databaseURL) - // FOR ALL TABLES mode disables the initial snapshot, so replication only - // sees rows written after the slot/publication exist. Insert continuously - // (rather than once upfront) so a row lands after startup completes. - writer := asyncroutine.NewPeriodic(100*time.Millisecond, func() { - _, err := db.Exec("INSERT INTO flights (name, created_at) VALUES ('alice', now());") - assert.NoError(t, err) - }) - writer.Start() - t.Cleanup(writer.Stop) + conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) + require.NoError(t, err) + + mgr := service.MockResources() + license.InjectTestService(mgr) + + input, err := newPgStreamInput(conf, mgr) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + // Bypass the benthos AsyncReader's infinite connect-retry loop, same as + // TestIntegrationNoSchemasMatchedReturnsError. + err = input.Connect(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "no schemas found matching pattern") +} + +// TestIntegrationMultiSchemaAutoDiscoverExcludeSchemas is +// TestIntegrationMultiSchemaExcludeSchemas with `tables` left unset: it +// verifies that leaving `tables` empty under schema_pattern auto-discovers +// the "events" table in each matched schema instead of falling back to a +// database-wide FOR ALL TABLES publication, so exclude_schemas still carves +// tenant_c out of both the snapshot and CDC. +func TestIntegrationMultiSchemaAutoDiscoverExcludeSchemas(t *testing.T) { + integration.CheckSkip(t) + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + // Three tenant schemas match tenant_*; tenant_c is carved out via exclude_schemas. + for _, schema := range []string{"tenant_a", "tenant_b", "tenant_c"} { + _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", schema)) + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf( + "CREATE TABLE %s.events (id SERIAL PRIMARY KEY, name TEXT)", schema)) + require.NoError(t, err) + } + + // Pre-load snapshot data, including a row in the excluded schema that must + // never surface. + _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('alice'), ('bob')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_b.events (name) VALUES ('carol')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_c.events (name) VALUES ('mallory')") + require.NoError(t, err) + + type msgMeta struct { + dbSchema string + table string + operation string + lsn string + } var ( mu sync.Mutex - collected []string + collected []msgMeta ) + collectedLen := func() int { + mu.Lock() + defer mu.Unlock() + return len(collected) + } + + // No `tables` field: every base table in tenant_a/tenant_b must be + // auto-discovered without listing "events" by hand. + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: auto_discover_exclude_schemas_test_slot + stream_snapshot: true + schema_pattern: tenant_* + exclude_schemas: + - tenant_c +`, databaseURL) sb := service.NewStreamBuilder() require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) @@ -2220,8 +2283,12 @@ postgres_cdc: mu.Lock() defer mu.Unlock() for _, msg := range batch { - table, _ := msg.MetaGet("table") - collected = append(collected, table) + m := msgMeta{} + m.dbSchema, _ = msg.MetaGet("database_schema") + m.table, _ = msg.MetaGet("table") + m.operation, _ = msg.MetaGet("operation") + m.lsn, _ = msg.MetaGet("lsn") + collected = append(collected, m) } return nil })) @@ -2236,17 +2303,72 @@ postgres_cdc: }() t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) - // Non-matching schema pattern must not block FOR ALL TABLES replication: - // the "flights" insert above should still arrive. + // Wait for the 3 snapshot rows from the two non-excluded schemas; tenant_c's + // row must never contribute to this count. assert.Eventually(t, func() bool { - mu.Lock() - defer mu.Unlock() - return len(collected) >= 1 - }, 30*time.Second, 100*time.Millisecond, "timed out waiting for FOR ALL TABLES replication; a non-matching schema pattern should be ignored when tables is empty") + return collectedLen() >= 3 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for snapshot rows") + + // Insert CDC rows into all three schemas, including the excluded one. + _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('dave')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_b.events (name) VALUES ('eve')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_c.events (name) VALUES ('trudy')") + require.NoError(t, err) + + // Wait for the 2 CDC rows from the non-excluded schemas (total 5). + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.Equal(c, 5, collectedLen()) + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for CDC rows") + + // tenant_c's CDC insert above raced the same replication stream as the + // tenant_a/tenant_b inserts already confirmed above, so if it were going + // to leak through it would have by now; assert the count never climbs + // past 5 to catch a delayed leak instead of just checking once. + assert.Never(t, func() bool { + return collectedLen() > 5 + }, 3*time.Second, 200*time.Millisecond, "received unexpected message(s) from excluded schema tenant_c") mu.Lock() defer mu.Unlock() - assert.Contains(t, collected, "flights") + + require.Len(t, collected, 5) + for _, m := range collected { + assert.NotEqual(t, "tenant_c", m.dbSchema, "tenant_c is excluded and must never appear, got message: %+v", m) + } + + var snapshots, cdcMsgs []msgMeta + for _, m := range collected { + if m.operation == "read" { + snapshots = append(snapshots, m) + } else { + cdcMsgs = append(cdcMsgs, m) + } + } + + // Snapshot assertions. + require.Len(t, snapshots, 3) + snapshotSchemas := make(map[string]int) + for _, m := range snapshots { + assert.Equal(t, "events", m.table, "snapshot: table should be bare name without schema prefix") + assert.Empty(t, m.lsn, "snapshot rows have no LSN") + snapshotSchemas[m.dbSchema]++ + } + assert.Equal(t, 2, snapshotSchemas["tenant_a"], "expected 2 snapshot rows from tenant_a") + assert.Equal(t, 1, snapshotSchemas["tenant_b"], "expected 1 snapshot row from tenant_b") + + // CDC assertions. + require.Len(t, cdcMsgs, 2) + cdcSchemas := make(map[string]int) + for _, m := range cdcMsgs { + assert.Equal(t, "insert", m.operation) + assert.Equal(t, "events", m.table) + assert.NotEmpty(t, m.lsn, "CDC rows must have an LSN") + cdcSchemas[m.dbSchema]++ + } + assert.Equal(t, 1, cdcSchemas["tenant_a"], "expected 1 CDC row from tenant_a") + assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } func TestIntegrationSchemaAndTableMatchingTest(t *testing.T) { diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 99439abd3f..29b5c47899 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -140,10 +140,10 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { schemas = remaining } - if len(inaccessibleSchemas) > 0 && len(config.DBTables) > 0 { + if len(inaccessibleSchemas) > 0 { config.Logger.Warnf("schema pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaPattern, inaccessibleSchemas) } - if len(schemas) == 0 && len(config.DBTables) > 0 { + if len(schemas) == 0 { return nil, fmt.Errorf("no schemas found matching pattern %q", config.DBSchemaPattern) } config.Logger.Infof("Schema pattern %q resolved to %d schema(s): %v", config.DBSchemaPattern, len(schemas), schemas) @@ -157,6 +157,14 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { normalizedTables = append(normalizedTables, normalized) } + // With no explicit table list, auto-discover every base table in each + // matched (and exclude_schemas-filtered) schema and publish them + // explicitly. Without this, an empty tables list would fall through to + // CreatePublication's FOR ALL TABLES fallback below, which replicates + // every schema in the database and silently defeats both schema_pattern + // and exclude_schemas. + autoDiscoverTables := len(normalizedTables) == 0 + tables = make([]TableFQN, 0, len(schemas)*len(normalizedTables)) foundTables := make(map[string]bool, len(normalizedTables)) for _, schema := range schemas { @@ -164,6 +172,12 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { if err != nil { return nil, fmt.Errorf("resolving tables in schema %q: %w", schema, err) } + if autoDiscoverTables { + for table := range existingTables { + tables = append(tables, TableFQN{Schema: schema, Table: table}) + } + continue + } for _, table := range normalizedTables { if _, ok := existingTables[table]; !ok { config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched pattern %q but does not contain this table)", schema, table, schema, config.DBSchemaPattern) @@ -173,18 +187,25 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { foundTables[table] = true } } - // A table must exist in at least one matched schema. Missing from some - // (but not all) matched schemas is tolerated above as a multi-tenant gap; - // missing from every matched schema is indistinguishable from a typo and - // must fail loudly rather than silently drop the table. - var missingTables []string - for i, table := range normalizedTables { - if !foundTables[table] { - missingTables = append(missingTables, config.DBTables[i]) + if autoDiscoverTables { + if len(tables) == 0 { + return nil, fmt.Errorf("no tables found in schema(s) %v matching pattern %q", schemas, config.DBSchemaPattern) + } + config.Logger.Infof("%q has no `tables` list configured: auto-discovered %d table(s) across %d schema(s)", config.DBSchemaPattern, len(tables), len(schemas)) + } else { + // A table must exist in at least one matched schema. Missing from some + // (but not all) matched schemas is tolerated above as a multi-tenant gap; + // missing from every matched schema is indistinguishable from a typo and + // must fail loudly rather than silently drop the table. + var missingTables []string + for i, table := range normalizedTables { + if !foundTables[table] { + missingTables = append(missingTables, config.DBTables[i]) + } + } + if len(missingTables) > 0 { + return nil, fmt.Errorf("table(s) %v not found in any schema matching pattern %q", missingTables, config.DBSchemaPattern) } - } - if len(missingTables) > 0 { - return nil, fmt.Errorf("table(s) %v not found in any schema matching pattern %q", missingTables, config.DBSchemaPattern) } } else { var err error From 96b8d78fbb9080358bea7773e0f2e444f4012d8a Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Thu, 13 Aug 2026 10:10:04 +0300 Subject: [PATCH 38/48] postgres_cdc: rename schema_pattern/exclude_schemas to schema_include/schema_exclude --- CHANGELOG.md | 4 +- .../components/pages/inputs/postgres_cdc.adoc | 36 +++++----- internal/impl/postgresql/input_pg_stream.go | 66 ++++++++--------- .../impl/postgresql/input_pg_stream_test.go | 54 +++++++------- internal/impl/postgresql/integration_test.go | 72 +++++++++---------- .../impl/postgresql/pglogicalstream/config.go | 14 ++-- .../pglogicalstream/logical_stream.go | 36 +++++----- .../pglogicalstream/schema_resolver.go | 6 +- .../postgresql/tests/current/Taskfile.yaml | 20 +++--- .../impl/postgresql/tests/current/setup.sql | 6 +- .../postgresql/tests/current/test_config.yaml | 4 +- ...est_config_schema_exclude_no_include.yaml} | 6 +- 12 files changed, 162 insertions(+), 162 deletions(-) rename internal/impl/postgresql/tests/current/{test_config_exclude_no_pattern.yaml => test_config_schema_exclude_no_include.yaml} (76%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1624384605..3196a0dc19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,8 +92,8 @@ All notable changes to this project will be documented in this file. ### Added -- postgres_cdc: Added a new `schema_pattern` field accepting a glob pattern (e.g. `tenant_*`), replicating all matching schemas through a single replication slot. Useful for multi-tenant databases where each tenant has its own schema. The existing `schema` field is unaffected and continues to take a single exact schema name (defaulting to `public`); `schema` and `schema_pattern` are mutually exclusive. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) -- postgres_cdc: Added a new `exclude_schemas` field to carve exceptions out of a broad `schema_pattern` (e.g. `schema_pattern: tenant_*` while skipping `tenant_test`). Accepts the same exact-name/glob/quoted syntax as `schema_pattern`, matches entries against the already-resolved schema list in memory with no extra database round-trips, and requires `schema_pattern` to be set. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) +- postgres_cdc: Added a new `schema_include` field accepting a glob pattern (e.g. `tenant_*`), replicating all matching schemas through a single replication slot. Useful for multi-tenant databases where each tenant has its own schema. Leaving `tables` unset auto-discovers every table in each matched schema instead of listing them by hand. The existing `schema` field is unaffected and continues to take a single exact schema name (defaulting to `public`); `schema` and `schema_include` are mutually exclusive. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) +- postgres_cdc: Added a new `schema_exclude` field to carve exceptions out of a broad `schema_include` (e.g. `schema_include: tenant_*` while skipping `tenant_test`). Accepts the same exact-name/glob/quoted syntax as `schema_include`, matches entries against the already-resolved schema list in memory with no extra database round-trips, and requires `schema_include` to be set. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) - aws_dynamodb_cdc: DynamoDB CDC now supports an optional checkpoint_namespace field, allowing multiple independent pipelines to share a single checkpoint table without overwriting each other's checkpoints. ([@squiidz](https://github.com/squiidz), [#4602](https://github.com/redpanda-data/connect/pull/4602)) ### Fixed diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 1539b17d55..3d4b3470f8 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -44,8 +44,8 @@ input: stream_snapshot: false snapshot_batch_size: 1000 schema: public - schema_pattern: "" - exclude_schemas: [] + schema_include: "" + schema_exclude: [] tables: [] # No default (required) checkpoint_limit: 1024 temporary_slot: false @@ -76,8 +76,8 @@ input: stream_snapshot: false snapshot_batch_size: 1000 schema: public - schema_pattern: "" - exclude_schemas: [] + schema_include: "" + schema_exclude: [] tables: [] # No default (required) checkpoint_limit: 1024 temporary_slot: false @@ -145,7 +145,7 @@ When set to true, empty messages with operation types BEGIN and COMMIT are gener === `stream_snapshot` -When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized. Note that this has no effect if `tables` is left empty and `schema_pattern` is NOT set, since in that case the snapshot is only planned for tables listed in `tables`. When `schema_pattern` IS set, leaving `tables` empty auto-discovers tables to snapshot instead - see `tables` below - and every discovered table must have a primary key. +When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized. Note that this has no effect if `tables` is left empty and `schema_include` is NOT set, since in that case the snapshot is only planned for tables listed in `tables`. When `schema_include` IS set, leaving `tables` empty auto-discovers tables to snapshot instead - see `tables` below - and every discovered table must have a primary key. *Type*: `bool` @@ -190,7 +190,7 @@ schema: public schema: '"MyCaseSensitiveSchemaNeedingQuotes"' ``` -=== `schema_pattern` +=== `schema_include` The PostgreSQL schema pattern to replicate data from. Accepts an exact schema name or a glob pattern using `*` as a wildcard to match multiple schemas. @@ -214,20 +214,20 @@ This field is mutually exclusive with `schema`; when set, it takes over schema r ```yml # Examples -schema_pattern: tenant_* +schema_include: tenant_* -schema_pattern: '*' +schema_include: '*' -schema_pattern: '"MyCaseSensitiveSchemaNeedingQuotes"' +schema_include: '"MyCaseSensitiveSchemaNeedingQuotes"' ``` -=== `exclude_schemas` +=== `schema_exclude` -A list of schema names or glob patterns to exclude from the schemas matched by `schema_pattern`. Only valid when `schema_pattern` is set. +A list of schema names or glob patterns to exclude from the schemas matched by `schema_include`. Only valid when `schema_include` is set. -Each entry uses the same syntax as `schema_pattern`: an exact schema name, a glob pattern using `*` as a wildcard, or a double-quoted exact identifier for an exact, case-sensitive match. +Each entry uses the same syntax as `schema_include`: an exact schema name, a glob pattern using `*` as a wildcard, or a double-quoted exact identifier for an exact, case-sensitive match. -A schema that matches `schema_pattern` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by `schema_pattern` is silently ignored, so a typo here simply excludes nothing rather than failing startup. +A schema that matches `schema_include` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by `schema_include` is silently ignored, so a typo here simply excludes nothing rather than failing startup. This exclusion is applied before `tables` is resolved, so it also takes effect when `tables` is left empty and tables are auto-discovered. @@ -239,7 +239,7 @@ This exclusion is applied before `tables` is resolved, so it also takes effect w ```yml # Examples -exclude_schemas: +schema_exclude: - tenant_internal - tenant_test_* ``` @@ -248,11 +248,11 @@ exclude_schemas: A list of table names to include in the logical replication. Each table should be specified as a separate item. -When `schema_pattern` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. +When `schema_include` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. -If left empty while `schema_pattern` is set, every base table in each matched (and un-excluded, see `exclude_schemas`) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. +If left empty while `schema_include` is set, every base table in each matched (and un-excluded, see `schema_exclude`) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. -If left empty while `schema_pattern` is NOT set, the underlying PostgreSQL publication is instead created `FOR ALL TABLES`, which replicates every table in every schema of the database, ignoring `schema`. This also disables `stream_snapshot`, since the initial snapshot is only planned for tables listed here. +If left empty while `schema_include` is NOT set, the underlying PostgreSQL publication is instead created `FOR ALL TABLES`, which replicates every table in every schema of the database, ignoring `schema`. This also disables `stream_snapshot`, since the initial snapshot is only planned for tables listed here. *Type*: `array` @@ -629,7 +629,7 @@ Optional external ID for the role assumption. === `signal_table_name` -The name of the table used to send control signals to the connector, excluding the schema. Not supported when `schema_pattern` is set, since there is no single schema to anchor the signal table to. The table must +The name of the table used to send control signals to the connector, excluding the schema. Not supported when `schema_include` is set, since there is no single schema to anchor the signal table to. The table must exist in the schema configured via the `schema` field, and must not also appear in `tables` — the signal table is implicitly added to the publication and excluded from snapshot scans, so listing it in both places is rejected at startup. It must have at least these columns — startup validation checks diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 9ffda49895..9472afb964 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -38,8 +38,8 @@ const ( fieldSnapshotMemSafetyFactor = "snapshot_memory_safety_factor" fieldSnapshotBatchSize = "snapshot_batch_size" fieldSchema = "schema" - fieldSchemaPattern = "schema_pattern" - fieldExcludeSchemas = "exclude_schemas" + fieldSchemaInclude = "schema_include" + fieldSchemaExclude = "schema_exclude" fieldTables = "tables" fieldCheckpointLimit = "checkpoint_limit" fieldTemporarySlot = "temporary_slot" @@ -104,7 +104,7 @@ This input adds the following metadata fields to each message: ShortDescription("Emit empty BEGIN and COMMIT messages at the start and end of each transaction."). Default(false)). Field(service.NewBoolField(fieldStreamSnapshot). - Description("When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized. Note that this has no effect if `" + fieldTables + "` is left empty and `" + fieldSchemaPattern + "` is NOT set, since in that case the snapshot is only planned for tables listed in `" + fieldTables + "`. When `" + fieldSchemaPattern + "` IS set, leaving `" + fieldTables + "` empty auto-discovers tables to snapshot instead - see `" + fieldTables + "` below - and every discovered table must have a primary key."). + Description("When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized. Note that this has no effect if `" + fieldTables + "` is left empty and `" + fieldSchemaInclude + "` is NOT set, since in that case the snapshot is only planned for tables listed in `" + fieldTables + "`. When `" + fieldSchemaInclude + "` IS set, leaving `" + fieldTables + "` empty auto-discovers tables to snapshot instead - see `" + fieldTables + "` below - and every discovered table must have a primary key."). ShortDescription("Stream a snapshot of all existing data before streaming changes. Snapshot tables must have a primary key."). Example(true). Default(false)). @@ -124,7 +124,7 @@ This input adds the following metadata fields to each message: Optional(). Default("public"), ). - Field(service.NewStringField(fieldSchemaPattern). + Field(service.NewStringField(fieldSchemaInclude). Description(`The PostgreSQL schema pattern to replicate data from. Accepts an exact schema name or a glob pattern using `+"`*`"+` as a wildcard to match multiple schemas. When a pattern is used, all schemas whose names match the pattern are replicated using a single replication slot and publication. This is useful for multi-tenant databases where each tenant has its own schema (e.g. `+"`tenant_*`"+` matches `+"`tenant_foo`"+`, `+"`tenant_bar`"+`, etc.). @@ -142,12 +142,12 @@ This field is mutually exclusive with `+"`"+fieldSchema+"`"+`; when set, it take Optional(). Default(""), ). - Field(service.NewStringListField(fieldExcludeSchemas). - Description(`A list of schema names or glob patterns to exclude from the schemas matched by ` + "`" + fieldSchemaPattern + "`" + `. Only valid when ` + "`" + fieldSchemaPattern + "`" + ` is set. + Field(service.NewStringListField(fieldSchemaExclude). + Description(`A list of schema names or glob patterns to exclude from the schemas matched by ` + "`" + fieldSchemaInclude + "`" + `. Only valid when ` + "`" + fieldSchemaInclude + "`" + ` is set. -Each entry uses the same syntax as ` + "`" + fieldSchemaPattern + "`" + `: an exact schema name, a glob pattern using ` + "`*`" + ` as a wildcard, or a double-quoted exact identifier for an exact, case-sensitive match. +Each entry uses the same syntax as ` + "`" + fieldSchemaInclude + "`" + `: an exact schema name, a glob pattern using ` + "`*`" + ` as a wildcard, or a double-quoted exact identifier for an exact, case-sensitive match. -A schema that matches ` + "`" + fieldSchemaPattern + "`" + ` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by ` + "`" + fieldSchemaPattern + "`" + ` is silently ignored, so a typo here simply excludes nothing rather than failing startup. +A schema that matches ` + "`" + fieldSchemaInclude + "`" + ` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by ` + "`" + fieldSchemaInclude + "`" + ` is silently ignored, so a typo here simply excludes nothing rather than failing startup. This exclusion is applied before ` + "`" + fieldTables + "`" + ` is resolved, so it also takes effect when ` + "`" + fieldTables + "`" + ` is left empty and tables are auto-discovered.`). Examples([]string{"tenant_internal", "tenant_test_*"}). @@ -157,11 +157,11 @@ This exclusion is applied before ` + "`" + fieldTables + "`" + ` is resolved, so Field(service.NewStringListField(fieldTables). Description(`A list of table names to include in the logical replication. Each table should be specified as a separate item. -When ` + "`schema_pattern`" + ` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. +When ` + "`" + fieldSchemaInclude + "`" + ` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. -If left empty while ` + "`schema_pattern`" + ` is set, every base table in each matched (and un-excluded, see ` + "`" + fieldExcludeSchemas + "`" + `) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. +If left empty while ` + "`" + fieldSchemaInclude + "`" + ` is set, every base table in each matched (and un-excluded, see ` + "`" + fieldSchemaExclude + "`" + `) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. -If left empty while ` + "`schema_pattern`" + ` is NOT set, the underlying PostgreSQL publication is instead created ` + "`FOR ALL TABLES`" + `, which replicates every table in every schema of the database, ignoring ` + "`" + fieldSchema + "`" + `. This also disables ` + "`" + fieldStreamSnapshot + "`" + `, since the initial snapshot is only planned for tables listed here.`). +If left empty while ` + "`" + fieldSchemaInclude + "`" + ` is NOT set, the underlying PostgreSQL publication is instead created ` + "`FOR ALL TABLES`" + `, which replicates every table in every schema of the database, ignoring ` + "`" + fieldSchema + "`" + `. This also disables ` + "`" + fieldStreamSnapshot + "`" + `, since the initial snapshot is only planned for tables listed here.`). Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`})). Field(service.NewIntField(fieldCheckpointLimit). Description("The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given LSN will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees."). @@ -251,7 +251,7 @@ This connector uses the naming pattern ` + "`pglog_stream_ 0 { - if schemaPattern == "" { - return nil, errors.New("exclude_schemas requires schema_pattern to be set") + if len(schemaExclude) > 0 { + if schemaInclude == "" { + return nil, errors.New("schema_exclude requires schema_include to be set") } - for i, pattern := range excludeSchemas { + for i, pattern := range schemaExclude { if err = validateSchemaPattern(pattern); err != nil { - return nil, fmt.Errorf("invalid exclude_schemas entry %q: %w", pattern, err) + return nil, fmt.Errorf("invalid schema_exclude entry %q: %w", pattern, err) } - // Normalize unquoted patterns to lower-case, mirroring schema_pattern + // Normalize unquoted patterns to lower-case, mirroring schema_include // above: PostgreSQL folds unquoted identifiers at creation time, so // TENANT_TEST and tenant_test resolve to the same schema. if !strings.HasPrefix(pattern, `"`) { - excludeSchemas[i] = strings.ToLower(pattern) + schemaExclude[i] = strings.ToLower(pattern) } } } @@ -437,8 +437,8 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser } if signalTableName != "" { - if schemaPattern != "" { - return nil, fmt.Errorf("%s is not supported when %s is set", fieldSignalTableName, fieldSchemaPattern) + if schemaInclude != "" { + return nil, fmt.Errorf("%s is not supported when %s is set", fieldSignalTableName, fieldSchemaInclude) } normalizedSignalTable, err := sanitize.NormalizePostgresIdentifier(signalTableName) if err != nil { @@ -492,8 +492,8 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser TLSConfig: pgConnConfig.TLSConfig, DBRawDSN: dsn, DBSchema: schema, - DBSchemaPattern: schemaPattern, - DBExcludeSchemas: excludeSchemas, + DBSchemaInclude: schemaInclude, + DBSchemaExclude: schemaExclude, DBTables: tables, RefreshAuthToken: iamAuthTokenBuilder, diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index fe682e4cb9..275b34fee4 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -44,12 +44,12 @@ tables: require.NoError(t, err) } -// TestSchemaPatternValidation verifies that the schema_pattern field is +// TestSchemaIncludeValidation verifies that the schema_include field is // validated during config parsing, before any network I/O is attempted. // Success is asserted via newPgStreamInput returning no error - the // constructor doesn't dial the database, so a valid pattern implies // validation passed. -func TestSchemaPatternValidation(t *testing.T) { +func TestSchemaIncludeValidation(t *testing.T) { tests := []struct { pattern string errContains string @@ -94,7 +94,7 @@ func TestSchemaPatternValidation(t *testing.T) { // quotes, e.g. `"MySchema"`) reaches validateSchemaPattern verbatim. yaml := fmt.Sprintf(` dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable -schema_pattern: '%s' +schema_include: '%s' slot_name: test_slot tables: - events @@ -111,30 +111,30 @@ tables: } } -// TestSchemaAndSchemaPatternMutuallyExclusive verifies that setting both -// schema (to a non-default value) and schema_pattern is rejected at config +// TestSchemaAndSchemaIncludeMutuallyExclusive verifies that setting both +// schema (to a non-default value) and schema_include is rejected at config // construction time. -func TestSchemaAndSchemaPatternMutuallyExclusive(t *testing.T) { +func TestSchemaAndSchemaIncludeMutuallyExclusive(t *testing.T) { yaml := ` dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable schema: tenant_foo -schema_pattern: 'tenant_*' +schema_include: 'tenant_*' slot_name: test_slot tables: - events ` _, err := parsePgStreamInput(t, yaml) require.Error(t, err) - assert.Contains(t, err.Error(), "schema and schema_pattern are mutually exclusive") + assert.Contains(t, err.Error(), "schema and schema_include are mutually exclusive") } -// TestSchemaPatternWithDefaultSchemaSucceeds verifies that setting -// schema_pattern while leaving schema untouched (at its "public" default) is +// TestSchemaIncludeWithDefaultSchemaSucceeds verifies that setting +// schema_include while leaving schema untouched (at its "public" default) is // allowed. -func TestSchemaPatternWithDefaultSchemaSucceeds(t *testing.T) { +func TestSchemaIncludeWithDefaultSchemaSucceeds(t *testing.T) { yaml := ` dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable -schema_pattern: 'tenant_*' +schema_include: 'tenant_*' slot_name: test_slot tables: - events @@ -143,11 +143,11 @@ tables: require.NoError(t, err) } -// TestExcludeSchemasValidation verifies that each exclude_schemas entry is -// validated with the same rules as schema_pattern - validateSchemaPattern is +// TestSchemaExcludeValidation verifies that each schema_exclude entry is +// validated with the same rules as schema_include - validateSchemaPattern is // reused rather than re-derived, so this exercises the same error cases -// TestSchemaPatternValidation covers, just reached through a different field. -func TestExcludeSchemasValidation(t *testing.T) { +// TestSchemaIncludeValidation covers, just reached through a different field. +func TestSchemaExcludeValidation(t *testing.T) { tests := []struct { pattern string errContains string @@ -166,8 +166,8 @@ func TestExcludeSchemasValidation(t *testing.T) { // quotes, e.g. `"MySchema"`) reaches validateSchemaPattern verbatim. yaml := fmt.Sprintf(` dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable -schema_pattern: 'tenant_*' -exclude_schemas: ['%s'] +schema_include: 'tenant_*' +schema_exclude: ['%s'] slot_name: test_slot tables: - events @@ -184,28 +184,28 @@ tables: } } -// TestExcludeSchemasRequiresSchemaPattern verifies that exclude_schemas is -// rejected at config-parse time when schema_pattern is left unset. Both +// TestSchemaExcludeRequiresSchemaInclude verifies that schema_exclude is +// rejected at config-parse time when schema_include is left unset. Both // single-exact-schema mode and FOR ALL TABLES mode (empty tables) have no // well-defined candidate set to exclude from, so this is a hard error rather // than a silent no-op. -func TestExcludeSchemasRequiresSchemaPattern(t *testing.T) { +func TestSchemaExcludeRequiresSchemaInclude(t *testing.T) { yaml := ` dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable -exclude_schemas: [tenant_test] +schema_exclude: [tenant_test] slot_name: test_slot tables: - events ` _, err := parsePgStreamInput(t, yaml) require.Error(t, err) - assert.Contains(t, err.Error(), "exclude_schemas requires schema_pattern to be set") + assert.Contains(t, err.Error(), "schema_exclude requires schema_include to be set") } -// TestExcludeSchemasEmptyWithoutSchemaPatternSucceeds verifies that leaving -// exclude_schemas at its default empty list does not trip the -// requires-schema_pattern check, since there's nothing to exclude. -func TestExcludeSchemasEmptyWithoutSchemaPatternSucceeds(t *testing.T) { +// TestSchemaExcludeEmptyWithoutSchemaIncludeSucceeds verifies that leaving +// schema_exclude at its default empty list does not trip the +// requires-schema_include check, since there's nothing to exclude. +func TestSchemaExcludeEmptyWithoutSchemaIncludeSucceeds(t *testing.T) { yaml := ` dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable slot_name: test_slot diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 07bcf48958..b88c2d6e8a 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1658,7 +1658,7 @@ postgres_cdc: dsn: %s slot_name: multi_schema_test_slot stream_snapshot: true - schema_pattern: tenant_* + schema_include: tenant_* tables: - events `, databaseURL) @@ -1746,17 +1746,17 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } -// TestIntegrationMultiSchemaExcludeSchemas verifies that exclude_schemas -// carves an exception out of a broad schema_pattern: a schema that matches -// schema_pattern but also matches an exclude_schemas entry contributes no +// TestIntegrationSchemaExcludeCarvesOutTenant verifies that schema_exclude +// carves an exception out of a broad schema_include: a schema that matches +// schema_include but also matches a schema_exclude entry contributes no // rows at all, neither during the initial snapshot nor from subsequent CDC // changes. -func TestIntegrationMultiSchemaExcludeSchemas(t *testing.T) { +func TestIntegrationSchemaExcludeCarvesOutTenant(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) - // Three tenant schemas match tenant_*; tenant_c is carved out via exclude_schemas. + // Three tenant schemas match tenant_*; tenant_c is carved out via schema_exclude. for _, schema := range []string{"tenant_a", "tenant_b", "tenant_c"} { _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", schema)) require.NoError(t, err) @@ -1794,10 +1794,10 @@ func TestIntegrationMultiSchemaExcludeSchemas(t *testing.T) { tmpl := fmt.Sprintf(` postgres_cdc: dsn: %s - slot_name: exclude_schemas_test_slot + slot_name: schema_exclude_test_slot stream_snapshot: true - schema_pattern: tenant_* - exclude_schemas: + schema_include: tenant_* + schema_exclude: - tenant_c tables: - events @@ -1898,7 +1898,7 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } -func TestIntegrationSchemaPatternMatchesHyphenatedUUIDSchema(t *testing.T) { +func TestIntegrationSchemaIncludeMatchesHyphenatedUUIDSchema(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) @@ -1930,9 +1930,9 @@ func TestIntegrationSchemaPatternMatchesHyphenatedUUIDSchema(t *testing.T) { tmpl := fmt.Sprintf(` postgres_cdc: dsn: %s - slot_name: hyphenated_schema_pattern_slot + slot_name: hyphenated_schema_include_slot stream_snapshot: true - schema_pattern: a0eebc99-* + schema_include: a0eebc99-* tables: - events `, databaseURL) @@ -2018,7 +2018,7 @@ postgres_cdc: dsn: %s slot_name: missing_table_degrade_slot stream_snapshot: true - schema_pattern: tenant_* + schema_include: tenant_* tables: - events `, databaseURL) @@ -2103,7 +2103,7 @@ postgres_cdc: dsn: %s slot_name: view_namesake_degrade_slot stream_snapshot: true - schema_pattern: tenant_* + schema_include: tenant_* tables: - events `, databaseURL) @@ -2156,7 +2156,7 @@ func TestIntegrationNoSchemasMatchedReturnsError(t *testing.T) { tmpl := fmt.Sprintf(` dsn: %s slot_name: no_schema_match_slot -schema_pattern: nonexistent_schema_zzz_* +schema_include: nonexistent_schema_zzz_* tables: - events `, databaseURL) @@ -2174,21 +2174,21 @@ tables: defer cancel() // Bypass the benthos AsyncReader's infinite connect-retry loop by calling - // Connect directly: a schema-pattern-not-found error is permanent, but + // Connect directly: a schema-include-not-found error is permanent, but // stream.Run has no path to surface it (it only returns once ctx is done), // so going through StreamBuilder/Run here would just time out instead. err = input.Connect(ctx) require.Error(t, err) - assert.Contains(t, err.Error(), "no schemas found matching pattern") + assert.Contains(t, err.Error(), "no schemas found matching schema_include pattern") } -// TestIntegrationSchemaPatternNonMatchingFailsEvenWithEmptyTables guards the -// fixed behaviour: schema_pattern always scopes replication, even when -// `tables` is left empty. A schema_pattern matching nothing in the database +// TestIntegrationSchemaIncludeNonMatchingFailsEvenWithEmptyTables guards the +// fixed behaviour: schema_include always scopes replication, even when +// `tables` is left empty. A schema_include matching nothing in the database // must fail startup rather than silently falling back to a database-wide -// FOR ALL TABLES publication (the old behaviour, which made exclude_schemas +// FOR ALL TABLES publication (the old behaviour, which made schema_exclude // meaningless whenever tables was left unset). -func TestIntegrationSchemaPatternNonMatchingFailsEvenWithEmptyTables(t *testing.T) { +func TestIntegrationSchemaIncludeNonMatchingFailsEvenWithEmptyTables(t *testing.T) { integration.CheckSkip(t) databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) @@ -2196,7 +2196,7 @@ func TestIntegrationSchemaPatternNonMatchingFailsEvenWithEmptyTables(t *testing. tmpl := fmt.Sprintf(` dsn: %s slot_name: no_schema_match_empty_tables_slot -schema_pattern: nonexistent_schema_zzz_* +schema_include: nonexistent_schema_zzz_* `, databaseURL) conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) @@ -2215,21 +2215,21 @@ schema_pattern: nonexistent_schema_zzz_* // TestIntegrationNoSchemasMatchedReturnsError. err = input.Connect(ctx) require.Error(t, err) - assert.Contains(t, err.Error(), "no schemas found matching pattern") + assert.Contains(t, err.Error(), "no schemas found matching schema_include pattern") } -// TestIntegrationMultiSchemaAutoDiscoverExcludeSchemas is -// TestIntegrationMultiSchemaExcludeSchemas with `tables` left unset: it -// verifies that leaving `tables` empty under schema_pattern auto-discovers +// TestIntegrationSchemaExcludeCarvesOutTenantAutoDiscover is +// TestIntegrationSchemaExcludeCarvesOutTenant with `tables` left unset: it +// verifies that leaving `tables` empty under schema_include auto-discovers // the "events" table in each matched schema instead of falling back to a -// database-wide FOR ALL TABLES publication, so exclude_schemas still carves +// database-wide FOR ALL TABLES publication, so schema_exclude still carves // tenant_c out of both the snapshot and CDC. -func TestIntegrationMultiSchemaAutoDiscoverExcludeSchemas(t *testing.T) { +func TestIntegrationSchemaExcludeCarvesOutTenantAutoDiscover(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) - // Three tenant schemas match tenant_*; tenant_c is carved out via exclude_schemas. + // Three tenant schemas match tenant_*; tenant_c is carved out via schema_exclude. for _, schema := range []string{"tenant_a", "tenant_b", "tenant_c"} { _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", schema)) require.NoError(t, err) @@ -2269,10 +2269,10 @@ func TestIntegrationMultiSchemaAutoDiscoverExcludeSchemas(t *testing.T) { tmpl := fmt.Sprintf(` postgres_cdc: dsn: %s - slot_name: auto_discover_exclude_schemas_test_slot + slot_name: auto_discover_schema_exclude_test_slot stream_snapshot: true - schema_pattern: tenant_* - exclude_schemas: + schema_include: tenant_* + schema_exclude: - tenant_c `, databaseURL) @@ -2425,7 +2425,7 @@ tables: tmpl := fmt.Sprintf(` dsn: %s slot_name: glob_schema_total_miss_slot -schema_pattern: tenant_* +schema_include: tenant_* tables: - orders - ordres @@ -2479,7 +2479,7 @@ postgres_cdc: dsn: %s slot_name: glob_schema_partial_match_slot stream_snapshot: true - schema_pattern: tenant_* + schema_include: tenant_* tables: - orders - ordres @@ -2558,7 +2558,7 @@ postgres_cdc: dsn: %s slot_name: glob_schema_full_match_slot stream_snapshot: true - schema_pattern: tenant_* + schema_include: tenant_* tables: - orders - ordres diff --git a/internal/impl/postgresql/pglogicalstream/config.go b/internal/impl/postgresql/pglogicalstream/config.go index 42f1e4b52c..0398995081 100644 --- a/internal/impl/postgresql/pglogicalstream/config.go +++ b/internal/impl/postgresql/pglogicalstream/config.go @@ -25,16 +25,16 @@ type Config struct { DBRawDSN string TLSConfig *tls.Config DBSchema string - // DBSchemaPattern is the glob pattern used to replicate from multiple + // DBSchemaInclude is the glob pattern used to replicate from multiple // schemas at once, using '*' as a wildcard (e.g. "tenant_*", "*"). When // non-empty, it takes precedence over DBSchema and schemas are resolved // dynamically at stream creation time. - DBSchemaPattern string - // DBExcludeSchemas is a list of schema names or glob patterns (same syntax - // as DBSchemaPattern) excluded from the schemas resolved by - // DBSchemaPattern. Only meaningful when DBSchemaPattern is non-empty. - DBExcludeSchemas []string - DBTables []string + DBSchemaInclude string + // DBSchemaExclude is a list of schema names or glob patterns (same syntax + // as DBSchemaInclude) excluded from the schemas resolved by + // DBSchemaInclude. Only meaningful when DBSchemaInclude is non-empty. + DBSchemaExclude []string + DBTables []string // Refreshes short lived IAM auth token that is treated as a password RefreshAuthToken func(ctx context.Context) error // ReplicationSlotName is the name of the replication slot to use diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 29b5c47899..a2e9d69201 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -102,26 +102,26 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { tables []TableFQN schema string ) - if config.DBSchemaPattern != "" { + if config.DBSchemaInclude != "" { if config.SignalTableName != "" { - return nil, errors.New("signal_table_name is not supported when schema_pattern is set") + return nil, errors.New("signal_table_name is not supported when schema_include is set") } - schemas, inaccessibleSchemas, err := resolveSchemas(ctx, dbConn, config.DBSchemaPattern) + schemas, inaccessibleSchemas, err := resolveSchemas(ctx, dbConn, config.DBSchemaInclude) if err != nil { - return nil, fmt.Errorf("resolving schema pattern %q: %w", config.DBSchemaPattern, err) + return nil, fmt.Errorf("resolving schema_include pattern %q: %w", config.DBSchemaInclude, err) } - if len(config.DBExcludeSchemas) > 0 { + if len(config.DBSchemaExclude) > 0 { // Filtering happens entirely against the schemas slice we already // fetched above - no extra DB round-trips per exclude pattern. var excluded []string remaining := make([]string, 0, len(schemas)) for _, schema := range schemas { var isExcluded bool - for _, pattern := range config.DBExcludeSchemas { + for _, pattern := range config.DBSchemaExclude { matched, err := schemaMatchesExcludePattern(schema, pattern) if err != nil { - return nil, fmt.Errorf("evaluating exclude_schemas pattern %q against schema %q: %w", pattern, schema, err) + return nil, fmt.Errorf("evaluating schema_exclude pattern %q against schema %q: %w", pattern, schema, err) } if matched { isExcluded = true @@ -135,18 +135,18 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { remaining = append(remaining, schema) } if len(excluded) > 0 { - config.Logger.Infof("exclude_schemas %v excluded %d schema(s) %v from schema pattern %q; %d schema(s) remain: %v", config.DBExcludeSchemas, len(excluded), excluded, config.DBSchemaPattern, len(remaining), remaining) + config.Logger.Infof("schema_exclude %v excluded %d schema(s) %v from schema_include pattern %q; %d schema(s) remain: %v", config.DBSchemaExclude, len(excluded), excluded, config.DBSchemaInclude, len(remaining), remaining) } schemas = remaining } if len(inaccessibleSchemas) > 0 { - config.Logger.Warnf("schema pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaPattern, inaccessibleSchemas) + config.Logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaInclude, inaccessibleSchemas) } if len(schemas) == 0 { - return nil, fmt.Errorf("no schemas found matching pattern %q", config.DBSchemaPattern) + return nil, fmt.Errorf("no schemas found matching schema_include pattern %q", config.DBSchemaInclude) } - config.Logger.Infof("Schema pattern %q resolved to %d schema(s): %v", config.DBSchemaPattern, len(schemas), schemas) + config.Logger.Infof("schema_include pattern %q resolved to %d schema(s): %v", config.DBSchemaInclude, len(schemas), schemas) normalizedTables := make([]string, 0, len(config.DBTables)) for _, table := range config.DBTables { @@ -158,11 +158,11 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } // With no explicit table list, auto-discover every base table in each - // matched (and exclude_schemas-filtered) schema and publish them + // matched (and schema_exclude-filtered) schema and publish them // explicitly. Without this, an empty tables list would fall through to // CreatePublication's FOR ALL TABLES fallback below, which replicates - // every schema in the database and silently defeats both schema_pattern - // and exclude_schemas. + // every schema in the database and silently defeats both schema_include + // and schema_exclude. autoDiscoverTables := len(normalizedTables) == 0 tables = make([]TableFQN, 0, len(schemas)*len(normalizedTables)) @@ -180,7 +180,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } for _, table := range normalizedTables { if _, ok := existingTables[table]; !ok { - config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched pattern %q but does not contain this table)", schema, table, schema, config.DBSchemaPattern) + config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched schema_include pattern %q but does not contain this table)", schema, table, schema, config.DBSchemaInclude) continue } tables = append(tables, TableFQN{Schema: schema, Table: table}) @@ -189,9 +189,9 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } if autoDiscoverTables { if len(tables) == 0 { - return nil, fmt.Errorf("no tables found in schema(s) %v matching pattern %q", schemas, config.DBSchemaPattern) + return nil, fmt.Errorf("no tables found in schema(s) %v matching schema_include pattern %q", schemas, config.DBSchemaInclude) } - config.Logger.Infof("%q has no `tables` list configured: auto-discovered %d table(s) across %d schema(s)", config.DBSchemaPattern, len(tables), len(schemas)) + config.Logger.Infof("%q has no `tables` list configured: auto-discovered %d table(s) across %d schema(s)", config.DBSchemaInclude, len(tables), len(schemas)) } else { // A table must exist in at least one matched schema. Missing from some // (but not all) matched schemas is tolerated above as a multi-tenant gap; @@ -204,7 +204,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } } if len(missingTables) > 0 { - return nil, fmt.Errorf("table(s) %v not found in any schema matching pattern %q", missingTables, config.DBSchemaPattern) + return nil, fmt.Errorf("table(s) %v not found in any schema matching schema_include pattern %q", missingTables, config.DBSchemaInclude) } } } else { diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 668314767e..765404b785 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -220,12 +220,12 @@ func escapeLike(s string) string { // schemaMatchesExcludePattern reports whether quotedSchemaName - a schema // identifier in the same quoted form resolveSchemas returns - matches -// excludePattern, an exclude_schemas entry written in the same shape as a -// schema_pattern value (an exact name, a '*' glob, or a double-quoted exact +// excludePattern, a schema_exclude entry written in the same shape as a +// schema_include value (an exact name, a '*' glob, or a double-quoted exact // identifier). // // Matching happens entirely in memory against a schema list we've already -// resolved, unlike resolveSchemas which queries the database: exclude_schemas +// resolved, unlike resolveSchemas which queries the database: schema_exclude // only ever narrows a candidate set that's already been fetched, so there's // no reason to pay for another round-trip per exclude pattern. Semantics // mirror schemaPatternToLike without going through SQL: a quoted pattern is diff --git a/internal/impl/postgresql/tests/current/Taskfile.yaml b/internal/impl/postgresql/tests/current/Taskfile.yaml index 0b57a7d25d..da2e348d06 100644 --- a/internal/impl/postgresql/tests/current/Taskfile.yaml +++ b/internal/impl/postgresql/tests/current/Taskfile.yaml @@ -73,37 +73,37 @@ tasks: # ── Schema validation smoke test ────────────────────────────────────────────── test:invalid-schema: - desc: Confirm that a malformed schema_pattern is rejected at startup, before any DB connection is attempted + desc: Confirm that a malformed schema_include is rejected at startup, before any DB connection is attempted env: PG_DSN: '{{.PG_DSN}}' cmds: - | set +e go run ../../../../../cmd/redpanda-connect/main.go run \ - --set 'input.postgres_cdc.schema_pattern=1abc' \ + --set 'input.postgres_cdc.schema_include=1abc' \ ./test_config.yaml 2>&1 | head -5 echo "exit $?" - # Expects: "invalid schema_pattern" error printed, process exits non-zero. - # schema_pattern is validated eagerly in newPgStreamInput + # Expects: "invalid schema_include" error printed, process exits non-zero. + # schema_include is validated eagerly in newPgStreamInput # (validateSchemaPattern), unlike schema which is only checked later, # against the live DB connection, inside NewPgStream. - # Note: an empty schema_pattern is NOT an error - it means "unset", and + # Note: an empty schema_include is NOT an error - it means "unset", and # the connector falls back to the schema field (default "public"). - test:exclude-schemas-requires-pattern: - desc: Confirm exclude_schemas is rejected at startup when schema_pattern is left unset + test:schema-exclude-requires-include: + desc: Confirm schema_exclude is rejected at startup when schema_include is left unset env: PG_DSN: '{{.PG_DSN}}' cmds: - | set +e go run ../../../../../cmd/redpanda-connect/main.go run \ - ./test_config_exclude_no_pattern.yaml 2>&1 | head -5 + ./test_config_schema_exclude_no_include.yaml 2>&1 | head -5 echo "exit $?" - # Expects: "exclude_schemas requires schema_pattern to be set" error + # Expects: "schema_exclude requires schema_include to be set" error # printed, process exits non-zero. Uses a dedicated fixture file rather # than --set on test_config.yaml, since --set rejects an empty RHS - # ("foo=" -> "expected foo=bar syntax"), so schema_pattern can't be + # ("foo=" -> "expected foo=bar syntax"), so schema_include can't be # cleared that way. # ── Quick sanity ────────────────────────────────────────────────────────────── diff --git a/internal/impl/postgresql/tests/current/setup.sql b/internal/impl/postgresql/tests/current/setup.sql index 3c0c6264fd..d231c381f7 100644 --- a/internal/impl/postgresql/tests/current/setup.sql +++ b/internal/impl/postgresql/tests/current/setup.sql @@ -1,10 +1,10 @@ -- Multi-schema CDC test setup --- Tests: schema glob (tenant_*), exclude_schemas (tenant_c), database_schema +-- Tests: schema glob (tenant_*), schema_exclude (tenant_c), database_schema -- metadata, commit_ts_ms, before (update/delete) -- ── Tenant schemas ──────────────────────────────────────────────────────────── -- tenant_a and tenant_b are replicated; tenant_c matches the tenant_* glob in --- test_config.yaml but is carved out via exclude_schemas — its rows must +-- test_config.yaml but is carved out via schema_exclude — its rows must -- never appear in the pipeline output. CREATE SCHEMA IF NOT EXISTS tenant_a; @@ -41,7 +41,7 @@ ALTER TABLE tenant_c.events REPLICA IDENTITY FULL; -- ── Seed snapshot rows ──────────────────────────────────────────────────────── -- These are visible during the initial snapshot (stream_snapshot: true). --- tenant_c's row (mallory) must NOT appear in the output — see exclude_schemas +-- tenant_c's row (mallory) must NOT appear in the output — see schema_exclude -- in test_config.yaml. INSERT INTO tenant_a.events (name) VALUES ('alice'), ('bob'); diff --git a/internal/impl/postgresql/tests/current/test_config.yaml b/internal/impl/postgresql/tests/current/test_config.yaml index 534b11496e..b440817dd1 100644 --- a/internal/impl/postgresql/tests/current/test_config.yaml +++ b/internal/impl/postgresql/tests/current/test_config.yaml @@ -4,10 +4,10 @@ input: slot_name: multi_schema_test_slot stream_snapshot: true # Glob pattern: replicates both tenant_a and tenant_b via one slot. - schema_pattern: tenant_* + schema_include: tenant_* # tenant_c matches the glob above but is carved out here — its rows must # never appear in the output below, neither during snapshot nor CDC. - exclude_schemas: + schema_exclude: - tenant_c tables: - events diff --git a/internal/impl/postgresql/tests/current/test_config_exclude_no_pattern.yaml b/internal/impl/postgresql/tests/current/test_config_schema_exclude_no_include.yaml similarity index 76% rename from internal/impl/postgresql/tests/current/test_config_exclude_no_pattern.yaml rename to internal/impl/postgresql/tests/current/test_config_schema_exclude_no_include.yaml index 57c0caf983..014332ed82 100644 --- a/internal/impl/postgresql/tests/current/test_config_exclude_no_pattern.yaml +++ b/internal/impl/postgresql/tests/current/test_config_schema_exclude_no_include.yaml @@ -1,12 +1,12 @@ -# Negative-validation fixture for exclude_schemas: schema_pattern is +# Negative-validation fixture for schema_exclude: schema_include is # deliberately left unset. --set can't assign an empty string (the CLI # rejects "foo=" as "expected foo=bar syntax"), so this exists as its own # file instead of overriding test_config.yaml at run time. input: postgres_cdc: dsn: ${PG_DSN:postgres://postgres:postgres@localhost:5433/testdb?sslmode=disable} - slot_name: exclude_schemas_no_pattern_slot - exclude_schemas: + slot_name: schema_exclude_no_include_slot + schema_exclude: - tenant_c tables: - events From 9897bbdd19eb350f53d3b3aee021f095e317ba23 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Thu, 20 Aug 2026 13:41:58 +0100 Subject: [PATCH 39/48] postgres_cdc: update docs --- docs/modules/components/pages/inputs/postgres_cdc.adoc | 5 ++++- internal/impl/postgresql/input_pg_stream.go | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 3d4b3470f8..a849989785 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -198,7 +198,10 @@ When a pattern is used, all schemas whose names match the pattern are replicated Double-quoted identifiers are treated as exact names and do not support wildcards. -Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted. +Schema pattern matching is re-evaluated every time the connector connects or reconnects - including the automatic reconnects that follow a transient replication failure - not just once at pipeline startup. This has two consequences that are easy to miss: + +- A schema created after the pipeline started that matches the pattern is picked up on the next reconnect and its tables are added to the publication. However, because the replication slot already exists by then, those tables are treated as already caught up, so with `stream_snapshot` enabled the rows that existed in that schema before it was picked up are never snapshotted and are silently missing from the output - only changes made after the schema is picked up are streamed. +- A schema that is dropped, renamed, or loses its `USAGE` grant between reconnects stops matching and its tables are silently removed from the publication on the next reconnect. A warning is logged when a schema becomes inaccessible due to a lost `USAGE` grant, but that warning is about the schema being inaccessible - it is not logged for a dropped or renamed schema, and nothing is ever logged about the resulting publication drop itself. If this pattern matches no schema in the database, startup fails with an error - this holds whether or not `tables` is set. See `tables` below for what happens when it's left empty. diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 9472afb964..d71ac82a08 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -131,7 +131,10 @@ When a pattern is used, all schemas whose names match the pattern are replicated Double-quoted identifiers are treated as exact names and do not support wildcards. -Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted. +Schema pattern matching is re-evaluated every time the connector connects or reconnects - including the automatic reconnects that follow a transient replication failure - not just once at pipeline startup. This has two consequences that are easy to miss: + +- A schema created after the pipeline started that matches the pattern is picked up on the next reconnect and its tables are added to the publication. However, because the replication slot already exists by then, those tables are treated as already caught up, so with `+"`"+fieldStreamSnapshot+"`"+` enabled the rows that existed in that schema before it was picked up are never snapshotted and are silently missing from the output - only changes made after the schema is picked up are streamed. +- A schema that is dropped, renamed, or loses its `+"`USAGE`"+` grant between reconnects stops matching and its tables are silently removed from the publication on the next reconnect. A warning is logged when a schema becomes inaccessible due to a lost `+"`USAGE`"+` grant, but that warning is about the schema being inaccessible - it is not logged for a dropped or renamed schema, and nothing is ever logged about the resulting publication drop itself. If this pattern matches no schema in the database, startup fails with an error - this holds whether or not `+"`"+fieldTables+"`"+` is set. See `+"`"+fieldTables+"`"+` below for what happens when it's left empty. From 5a78e090690b6f78f38892c79763582a194590f0 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Thu, 20 Aug 2026 16:40:26 +0100 Subject: [PATCH 40/48] postgres_cdc: update tables docs --- docs/modules/components/pages/inputs/postgres_cdc.adoc | 5 +++-- internal/impl/postgresql/input_pg_stream.go | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index a849989785..1f419c285d 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -46,7 +46,7 @@ input: schema: public schema_include: "" schema_exclude: [] - tables: [] # No default (required) + tables: [] checkpoint_limit: 1024 temporary_slot: false slot_name: my_test_slot # No default (required) @@ -78,7 +78,7 @@ input: schema: public schema_include: "" schema_exclude: [] - tables: [] # No default (required) + tables: [] checkpoint_limit: 1024 temporary_slot: false slot_name: my_test_slot # No default (required) @@ -260,6 +260,7 @@ If left empty while `schema_include` is NOT set, the underlying PostgreSQL publi *Type*: `array` +*Default*: `[]` ```yml # Examples diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index d71ac82a08..becbe1f522 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -165,7 +165,9 @@ When ` + "`" + fieldSchemaInclude + "`" + ` is set, this list is resolved agains If left empty while ` + "`" + fieldSchemaInclude + "`" + ` is set, every base table in each matched (and un-excluded, see ` + "`" + fieldSchemaExclude + "`" + `) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. If left empty while ` + "`" + fieldSchemaInclude + "`" + ` is NOT set, the underlying PostgreSQL publication is instead created ` + "`FOR ALL TABLES`" + `, which replicates every table in every schema of the database, ignoring ` + "`" + fieldSchema + "`" + `. This also disables ` + "`" + fieldStreamSnapshot + "`" + `, since the initial snapshot is only planned for tables listed here.`). - Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`})). + Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`}). + Optional(). + Default([]string{})). Field(service.NewIntField(fieldCheckpointLimit). Description("The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given LSN will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees."). ShortDescription("The maximum number of messages that can be processed at a given time."). From bc81de9be3fb2c566b8998176479671610c240ea Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Thu, 20 Aug 2026 22:46:04 +0100 Subject: [PATCH 41/48] postgres_cdc: clean up testing and validation --- internal/impl/postgresql/integration_test.go | 134 +++++++++++------- .../pglogicalstream/logical_stream.go | 5 + 2 files changed, 90 insertions(+), 49 deletions(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index b88c2d6e8a..afd4d811fa 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1751,7 +1751,7 @@ postgres_cdc: // schema_include but also matches a schema_exclude entry contributes no // rows at all, neither during the initial snapshot nor from subsequent CDC // changes. -func TestIntegrationSchemaExcludeCarvesOutTenant(t *testing.T) { +func TestIntegrationMultiSchemaExcludeCarvesOutTenant(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) @@ -1898,7 +1898,7 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } -func TestIntegrationSchemaIncludeMatchesHyphenatedUUIDSchema(t *testing.T) { +func TestIntegrationMultiSchemaIncludeMatchesHyphenatedUUIDSchema(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) @@ -2148,12 +2148,14 @@ postgres_cdc: assert.Equal(t, "events", collected[0].table) } -func TestIntegrationNoSchemasMatchedReturnsError(t *testing.T) { +func TestIntegrationMultiSchemaIncludeExcludeConfigValidation(t *testing.T) { integration.CheckSkip(t) - databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") - require.NoError(t, err) - tmpl := fmt.Sprintf(` + t.Run("schema_include matches nothing", func(t *testing.T) { + databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + tmpl := fmt.Sprintf(` dsn: %s slot_name: no_schema_match_slot schema_include: nonexistent_schema_zzz_* @@ -2161,61 +2163,95 @@ tables: - events `, databaseURL) - conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) - require.NoError(t, err) + conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) + require.NoError(t, err) - mgr := service.MockResources() - license.InjectTestService(mgr) + mgr := service.MockResources() + license.InjectTestService(mgr) - input, err := newPgStreamInput(conf, mgr) - require.NoError(t, err) + input, err := newPgStreamInput(conf, mgr) + require.NoError(t, err) - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() - // Bypass the benthos AsyncReader's infinite connect-retry loop by calling - // Connect directly: a schema-include-not-found error is permanent, but - // stream.Run has no path to surface it (it only returns once ctx is done), - // so going through StreamBuilder/Run here would just time out instead. - err = input.Connect(ctx) - require.Error(t, err) - assert.Contains(t, err.Error(), "no schemas found matching schema_include pattern") -} + err = input.Connect(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "no schemas found matching schema_include pattern") + }) -// TestIntegrationSchemaIncludeNonMatchingFailsEvenWithEmptyTables guards the -// fixed behaviour: schema_include always scopes replication, even when -// `tables` is left empty. A schema_include matching nothing in the database -// must fail startup rather than silently falling back to a database-wide -// FOR ALL TABLES publication (the old behaviour, which made schema_exclude -// meaningless whenever tables was left unset). -func TestIntegrationSchemaIncludeNonMatchingFailsEvenWithEmptyTables(t *testing.T) { - integration.CheckSkip(t) - databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") - require.NoError(t, err) + t.Run("schema_include matches nothing with empty tables", func(t *testing.T) { + databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) - tmpl := fmt.Sprintf(` + tmpl := fmt.Sprintf(` dsn: %s slot_name: no_schema_match_empty_tables_slot schema_include: nonexistent_schema_zzz_* `, databaseURL) - conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) - require.NoError(t, err) + conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) + require.NoError(t, err) - mgr := service.MockResources() - license.InjectTestService(mgr) + mgr := service.MockResources() + license.InjectTestService(mgr) - input, err := newPgStreamInput(conf, mgr) - require.NoError(t, err) + input, err := newPgStreamInput(conf, mgr) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() + // Bypass the benthos AsyncReader's infinite connect-retry loop, same as + // the "schema_include matches nothing" case above. + err = input.Connect(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "no schemas found matching schema_include pattern") + }) - // Bypass the benthos AsyncReader's infinite connect-retry loop, same as - // TestIntegrationNoSchemasMatchedReturnsError. - err = input.Connect(ctx) - require.Error(t, err) - assert.Contains(t, err.Error(), "no schemas found matching schema_include pattern") + t.Run("schema_exclude excludes every matched schema", func(t *testing.T) { + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + // Both schemas match tenant_*, but schema_exclude below excludes them all. + for _, schema := range []string{"tenant_a", "tenant_b"} { + _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", schema)) + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf( + "CREATE TABLE %s.events (id SERIAL PRIMARY KEY, name TEXT)", schema)) + require.NoError(t, err) + } + + tmpl := fmt.Sprintf(` +dsn: %s +slot_name: schema_exclude_all_matched_slot +schema_include: tenant_* +schema_exclude: + - tenant_* +tables: + - events +`, databaseURL) + + conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) + require.NoError(t, err) + + mgr := service.MockResources() + license.InjectTestService(mgr) + + input, err := newPgStreamInput(conf, mgr) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + // Bypass the benthos AsyncReader's infinite connect-retry loop, same as + // the "schema_include matches nothing" case above. + err = input.Connect(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "matched schema(s)") + assert.Contains(t, err.Error(), "excluded all of them") + assert.NotContains(t, err.Error(), "no schemas found matching schema_include pattern") + }) } // TestIntegrationSchemaExcludeCarvesOutTenantAutoDiscover is @@ -2224,7 +2260,7 @@ schema_include: nonexistent_schema_zzz_* // the "events" table in each matched schema instead of falling back to a // database-wide FOR ALL TABLES publication, so schema_exclude still carves // tenant_c out of both the snapshot and CDC. -func TestIntegrationSchemaExcludeCarvesOutTenantAutoDiscover(t *testing.T) { +func TestIntegrationMultiSchemaExcludeCarvesOutTenantAutoDiscover(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) @@ -2371,7 +2407,7 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } -func TestIntegrationSchemaAndTableMatchingTest(t *testing.T) { +func TestIntegrationMultiSchemaAndTableMatchingTest(t *testing.T) { integration.CheckSkip(t) t.Run("exact schema match with missing table fails", func(t *testing.T) { @@ -2403,7 +2439,7 @@ tables: defer cancel() // Bypass the benthos AsyncReader's infinite connect-retry loop, same as - // TestIntegrationNoSchemasMatchedReturnsError. + // TestIntegrationSchemaIncludeExcludeConfigValidation. err = input.Connect(ctx) require.Error(t, err, "typo'd table %q should fail startup loudly instead of silently streaming only %q", "ordres", "orders") assert.Contains(t, err.Error(), "ordres") diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index a2e9d69201..d3436dad02 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -110,6 +110,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { if err != nil { return nil, fmt.Errorf("resolving schema_include pattern %q: %w", config.DBSchemaInclude, err) } + matchedSchemas := schemas if len(config.DBSchemaExclude) > 0 { // Filtering happens entirely against the schemas slice we already @@ -143,7 +144,11 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { if len(inaccessibleSchemas) > 0 { config.Logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaInclude, inaccessibleSchemas) } + if len(schemas) == 0 { + if len(matchedSchemas) > 0 { + return nil, fmt.Errorf("schema_include pattern %q matched schema(s) %v, but schema_exclude %v excluded all of them", config.DBSchemaInclude, matchedSchemas, config.DBSchemaExclude) + } return nil, fmt.Errorf("no schemas found matching schema_include pattern %q", config.DBSchemaInclude) } config.Logger.Infof("schema_include pattern %q resolved to %d schema(s): %v", config.DBSchemaInclude, len(schemas), schemas) From ad32e8e1be67a12194f4e0a50f77a0df4f9c0b01 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Thu, 20 Aug 2026 23:26:30 +0100 Subject: [PATCH 42/48] postgres_cdc: fix link --- internal/impl/postgresql/input_pg_stream_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index 275b34fee4..322de40f24 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -4,7 +4,7 @@ // License (the "License"); you may not use this file except in compliance with // the License. You may obtain a copy of the License at // -// https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md package pgstream From 6c0ffcb6d719421d03493ad9eba06daee29118cb Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Thu, 20 Aug 2026 23:44:05 +0100 Subject: [PATCH 43/48] fix: --- .../impl/postgresql/pglogicalstream/config.go | 4 +++ .../pglogicalstream/logical_stream.go | 14 ++++++++- .../pglogicalstream/schema_resolver.go | 29 +++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/internal/impl/postgresql/pglogicalstream/config.go b/internal/impl/postgresql/pglogicalstream/config.go index 0398995081..8804efcd5c 100644 --- a/internal/impl/postgresql/pglogicalstream/config.go +++ b/internal/impl/postgresql/pglogicalstream/config.go @@ -63,4 +63,8 @@ type Config struct { UnchangedToastValue any // The interval to send logical messages HeartbeatInterval time.Duration + + // Used to track and report on resolved schemas between reconnects + previouslyResolvedSchemas []string + previouslyInaccessibleSchemas []string } diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index d3436dad02..7ceea54d7b 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -141,9 +141,10 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { schemas = remaining } - if len(inaccessibleSchemas) > 0 { + if len(inaccessibleSchemas) > 0 && !slices.Equal(inaccessibleSchemas, config.previouslyInaccessibleSchemas) { config.Logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaInclude, inaccessibleSchemas) } + config.previouslyInaccessibleSchemas = slices.Clone(inaccessibleSchemas) if len(schemas) == 0 { if len(matchedSchemas) > 0 { @@ -153,6 +154,17 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } config.Logger.Infof("schema_include pattern %q resolved to %d schema(s): %v", config.DBSchemaInclude, len(schemas), schemas) + if config.previouslyResolvedSchemas != nil { + added, removed := diffSchemaSets(config.previouslyResolvedSchemas, schemas) + if len(added) > 0 { + config.Logger.Warnf("schema_include pattern %q now also matches schema(s) %v that did not match on the previous connect; their tables are being added to the publication, but any rows already in them will NOT be snapshotted even if stream_snapshot is enabled - only changes made from now on will be captured", config.DBSchemaInclude, added) + } + if len(removed) > 0 { + config.Logger.Warnf("schema(s) %v no longer match schema_include pattern %q (dropped, renamed, or the role lost USAGE) since the previous connect; their tables are being removed from the publication and will stop replicating", removed, config.DBSchemaInclude) + } + } + config.previouslyResolvedSchemas = slices.Clone(schemas) + normalizedTables := make([]string, 0, len(config.DBTables)) for _, table := range config.DBTables { normalized, err := sanitize.NormalizePostgresIdentifier(table) diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 765404b785..bbbbc061a7 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -256,6 +256,35 @@ func schemaMatchesExcludePattern(quotedSchemaName, excludePattern string) (bool, return re.MatchString(strings.ToLower(schemaName)), nil } +// diffSchemaSets reports which schemas are present in current but not +// previous (added) and vice versa (removed), used to detect schema-set drift +// between the schema_include resolution done on this connect and the one +// done on the previous connect/reconnect. A nil previous (the very first +// resolution in the process's lifetime) is not a meaningful "everything was +// just added" drift, so callers should only act on the result when previous +// is non-nil. +func diffSchemaSets(previous, current []string) (added, removed []string) { + previousSet := make(map[string]struct{}, len(previous)) + for _, schema := range previous { + previousSet[schema] = struct{}{} + } + currentSet := make(map[string]struct{}, len(current)) + for _, schema := range current { + currentSet[schema] = struct{}{} + } + for _, schema := range current { + if _, ok := previousSet[schema]; !ok { + added = append(added, schema) + } + } + for _, schema := range previous { + if _, ok := currentSet[schema]; !ok { + removed = append(removed, schema) + } + } + return added, removed +} + // globToRegexp compiles an unquoted glob pattern (using '*' as a wildcard) // into an anchored regexp - the in-memory equivalent of globToLike for // callers matching against values already held in Go rather than via a SQL From 129878a05151c524fbe7cba289915a4512c49efc Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Fri, 21 Aug 2026 02:14:06 +0100 Subject: [PATCH 44/48] postgres_cdc: move schema resolution into dedicated file --- .../pglogicalstream/logical_stream.go | 58 +--------------- .../pglogicalstream/schema_resolver.go | 68 +++++++++++++++++++ 2 files changed, 70 insertions(+), 56 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 7ceea54d7b..ddeaf31a9c 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -106,64 +106,10 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { if config.SignalTableName != "" { return nil, errors.New("signal_table_name is not supported when schema_include is set") } - schemas, inaccessibleSchemas, err := resolveSchemas(ctx, dbConn, config.DBSchemaInclude) + schemas, err := resolveIncludedSchemas(ctx, dbConn, config) if err != nil { - return nil, fmt.Errorf("resolving schema_include pattern %q: %w", config.DBSchemaInclude, err) - } - matchedSchemas := schemas - - if len(config.DBSchemaExclude) > 0 { - // Filtering happens entirely against the schemas slice we already - // fetched above - no extra DB round-trips per exclude pattern. - var excluded []string - remaining := make([]string, 0, len(schemas)) - for _, schema := range schemas { - var isExcluded bool - for _, pattern := range config.DBSchemaExclude { - matched, err := schemaMatchesExcludePattern(schema, pattern) - if err != nil { - return nil, fmt.Errorf("evaluating schema_exclude pattern %q against schema %q: %w", pattern, schema, err) - } - if matched { - isExcluded = true - break - } - } - if isExcluded { - excluded = append(excluded, schema) - continue - } - remaining = append(remaining, schema) - } - if len(excluded) > 0 { - config.Logger.Infof("schema_exclude %v excluded %d schema(s) %v from schema_include pattern %q; %d schema(s) remain: %v", config.DBSchemaExclude, len(excluded), excluded, config.DBSchemaInclude, len(remaining), remaining) - } - schemas = remaining - } - - if len(inaccessibleSchemas) > 0 && !slices.Equal(inaccessibleSchemas, config.previouslyInaccessibleSchemas) { - config.Logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaInclude, inaccessibleSchemas) - } - config.previouslyInaccessibleSchemas = slices.Clone(inaccessibleSchemas) - - if len(schemas) == 0 { - if len(matchedSchemas) > 0 { - return nil, fmt.Errorf("schema_include pattern %q matched schema(s) %v, but schema_exclude %v excluded all of them", config.DBSchemaInclude, matchedSchemas, config.DBSchemaExclude) - } - return nil, fmt.Errorf("no schemas found matching schema_include pattern %q", config.DBSchemaInclude) - } - config.Logger.Infof("schema_include pattern %q resolved to %d schema(s): %v", config.DBSchemaInclude, len(schemas), schemas) - - if config.previouslyResolvedSchemas != nil { - added, removed := diffSchemaSets(config.previouslyResolvedSchemas, schemas) - if len(added) > 0 { - config.Logger.Warnf("schema_include pattern %q now also matches schema(s) %v that did not match on the previous connect; their tables are being added to the publication, but any rows already in them will NOT be snapshotted even if stream_snapshot is enabled - only changes made from now on will be captured", config.DBSchemaInclude, added) - } - if len(removed) > 0 { - config.Logger.Warnf("schema(s) %v no longer match schema_include pattern %q (dropped, renamed, or the role lost USAGE) since the previous connect; their tables are being removed from the publication and will stop replicating", removed, config.DBSchemaInclude) - } + return nil, err } - config.previouslyResolvedSchemas = slices.Clone(schemas) normalizedTables := make([]string, 0, len(config.DBTables)) for _, table := range config.DBTables { diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index bbbbc061a7..dcc8df8bc2 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -12,6 +12,7 @@ import ( "context" "fmt" "regexp" + "slices" "strings" "github.com/jackc/pgx/v5/pgconn" @@ -130,6 +131,73 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (v return schemas, hidden, nil } +// resolveIncludedSchemas resolves config.DBSchemaInclude against conn, +// applies config.DBSchemaExclude filtering, and logs schema-set drift +// against the previous resolution cached on config (see +// Config.previouslyResolvedSchemas) +func resolveIncludedSchemas(ctx context.Context, conn *pgconn.PgConn, cfg *Config) ([]string, error) { + schemas, inaccessibleSchemas, err := resolveSchemas(ctx, conn, cfg.DBSchemaInclude) + if err != nil { + return nil, fmt.Errorf("resolving schema_include pattern %q: %w", cfg.DBSchemaInclude, err) + } + matchedSchemas := schemas + + if len(cfg.DBSchemaExclude) > 0 { + // Filtering happens entirely against the schemas slice we already + // fetched above - no extra DB round-trips per exclude pattern. + var excluded []string + remaining := make([]string, 0, len(schemas)) + for _, schema := range schemas { + var isExcluded bool + for _, pattern := range cfg.DBSchemaExclude { + matched, err := schemaMatchesExcludePattern(schema, pattern) + if err != nil { + return nil, fmt.Errorf("evaluating schema_exclude pattern %q against schema %q: %w", pattern, schema, err) + } + if matched { + isExcluded = true + break + } + } + if isExcluded { + excluded = append(excluded, schema) + continue + } + remaining = append(remaining, schema) + } + if len(excluded) > 0 { + cfg.Logger.Infof("schema_exclude %v excluded %d schema(s) %v from schema_include pattern %q; %d schema(s) remain: %v", cfg.DBSchemaExclude, len(excluded), excluded, cfg.DBSchemaInclude, len(remaining), remaining) + } + schemas = remaining + } + + if len(inaccessibleSchemas) > 0 && !slices.Equal(inaccessibleSchemas, cfg.previouslyInaccessibleSchemas) { + cfg.Logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", cfg.DBSchemaInclude, inaccessibleSchemas) + } + cfg.previouslyInaccessibleSchemas = slices.Clone(inaccessibleSchemas) + + if len(schemas) == 0 { + if len(matchedSchemas) > 0 { + return nil, fmt.Errorf("schema_include pattern %q matched schema(s) %v, but schema_exclude %v excluded all of them", cfg.DBSchemaInclude, matchedSchemas, cfg.DBSchemaExclude) + } + return nil, fmt.Errorf("no schemas found matching schema_include pattern %q", cfg.DBSchemaInclude) + } + cfg.Logger.Infof("schema_include pattern %q resolved to %d schema(s): %v", cfg.DBSchemaInclude, len(schemas), schemas) + + if cfg.previouslyResolvedSchemas != nil { + added, removed := diffSchemaSets(cfg.previouslyResolvedSchemas, schemas) + if len(added) > 0 { + cfg.Logger.Warnf("schema_include pattern %q now also matches schema(s) %v that did not match on the previous connect; their tables are being added to the publication, but any rows already in them will NOT be snapshotted even if stream_snapshot is enabled - only changes made from now on will be captured", cfg.DBSchemaInclude, added) + } + if len(removed) > 0 { + cfg.Logger.Warnf("schema(s) %v no longer match schema_include pattern %q (dropped, renamed, or the role lost USAGE) since the previous connect; their tables are being removed from the publication and will stop replicating", removed, cfg.DBSchemaInclude) + } + } + cfg.previouslyResolvedSchemas = slices.Clone(schemas) + + return schemas, nil +} + // resolveExistingTables returns the quoted names of the publishable base // tables (including partitioned tables) that exist in the given (already // quoted) schema. From ad1c1325707de76d1586122992e8129a9dbd9ab1 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Fri, 21 Aug 2026 02:28:19 +0100 Subject: [PATCH 45/48] clean up --- .../impl/postgresql/pglogicalstream/config.go | 22 ++-- .../pglogicalstream/schema_resolver.go | 107 +++++------------- .../plugins/cdctest/cdc_conformance_test.go | 21 +--- 3 files changed, 42 insertions(+), 108 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/config.go b/internal/impl/postgresql/pglogicalstream/config.go index 8804efcd5c..d8400b7c2f 100644 --- a/internal/impl/postgresql/pglogicalstream/config.go +++ b/internal/impl/postgresql/pglogicalstream/config.go @@ -25,16 +25,14 @@ type Config struct { DBRawDSN string TLSConfig *tls.Config DBSchema string - // DBSchemaInclude is the glob pattern used to replicate from multiple - // schemas at once, using '*' as a wildcard (e.g. "tenant_*", "*"). When - // non-empty, it takes precedence over DBSchema and schemas are resolved - // dynamically at stream creation time. - DBSchemaInclude string - // DBSchemaExclude is a list of schema names or glob patterns (same syntax - // as DBSchemaInclude) excluded from the schemas resolved by - // DBSchemaInclude. Only meaningful when DBSchemaInclude is non-empty. - DBSchemaExclude []string - DBTables []string + DBTables []string + + // Multi schema support + DBSchemaInclude string + DBSchemaExclude []string + previouslyResolvedSchemas []string // track and report on resolved schemas between reconnects + previouslyInaccessibleSchemas []string + // Refreshes short lived IAM auth token that is treated as a password RefreshAuthToken func(ctx context.Context) error // ReplicationSlotName is the name of the replication slot to use @@ -63,8 +61,4 @@ type Config struct { UnchangedToastValue any // The interval to send logical messages HeartbeatInterval time.Duration - - // Used to track and report on resolved schemas between reconnects - previouslyResolvedSchemas []string - previouslyInaccessibleSchemas []string } diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index dcc8df8bc2..eef25e3c89 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -21,19 +21,10 @@ import ( ) // schemaPatternToLike converts a schema name or glob pattern into the LIKE -// pattern used by resolveSchemas, plus whether that pattern must be matched -// case-sensitively. Extracted for unit testing. -// -// For quoted identifiers the inner name is exact-escaped (no wildcard -// expansion) and matched case-sensitively, since a quoted identifier's case -// is significant and PostgreSQL does not fold it. For unquoted patterns the -// '*' wildcard is converted to '%' and matching is case-insensitive: this -// mirrors PostgreSQL folding unquoted identifiers to lower-case at creation -// time for the common case, but must hold even when a schema had to be -// created with a quoted identifier for an unrelated reason (e.g. a -// UUID-suffixed tenant schema, which requires quoting because hyphens are -// invalid in unquoted identifiers) and so kept whatever case it was written -// with — an unquoted glob like "tenant_*" is still expected to match it. +// pattern used by resolveSchemas, plus whether it must be matched +// case-sensitively. Quoted identifiers are matched exactly and +// case-sensitively; unquoted patterns use '*' as a wildcard and match +// case-insensitively, mirroring how PostgreSQL folds unquoted identifiers. func schemaPatternToLike(pattern string) (likePattern string, caseSensitive bool, err error) { if strings.HasPrefix(pattern, `"`) { unquoted, err := sanitize.UnquotePostgresIdentifier(pattern) @@ -45,25 +36,13 @@ func schemaPatternToLike(pattern string) (likePattern string, caseSensitive bool return globToLike(strings.ToLower(pattern)), false, nil } -// resolveSchemas returns the schemas matching pattern that the connection's -// role has access to (visibleSchemas), plus any schemas that also match -// pattern but are hidden from information_schema.schemata by privileges -// (inaccessibleSchemas). The latter is surfaced separately so callers can -// warn the user instead of silently dropping schemas they expected to be -// included, e.g. `"tenant_*"` matching a schema the configured role can't -// see yet. -// -// For unquoted patterns (e.g. "tenant_*") the pattern is matched -// case-insensitively via ILIKE, regardless of whether the matched schema was -// itself created case-insensitively. For quoted identifiers (e.g. -// `"MySchema"`) an exact case-sensitive lookup is performed via LIKE. System -// schemas (pg_* and information_schema) are always excluded case-sensitively -// so that wildcard patterns like "*" do not attempt to replicate catalog -// tables. -// -// Returned schema names are quoted PostgreSQL identifiers. Returns an error -// if either query fails; returns a nil visibleSchemas slice (with a nil err) -// if no schemas match — callers should treat that as an error condition. +// resolveSchemas returns the schemas matching pattern that the role can see +// (visibleSchemas), plus schemas that also match but are hidden by missing +// privileges (inaccessibleSchemas) - surfaced separately so callers can warn +// instead of silently dropping them. System schemas (pg_* and +// information_schema) are always excluded. Returned names are quoted +// PostgreSQL identifiers. A nil visibleSchemas with a nil error means no +// match - callers should treat that as an error. func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (visibleSchemas, inaccessibleSchemas []string, err error) { likePattern, caseSensitive, err := schemaPatternToLike(pattern) if err != nil { @@ -102,9 +81,8 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (v } } - // pg_namespace is not privilege-filtered, so any pattern match here that's - // missing from information_schema.schemata means the role lacks USAGE (or - // similar) on that schema rather than the schema simply not existing. + // pg_namespace isn't privilege-filtered, so a match here missing from + // information_schema.schemata means the role lacks USAGE on that schema. nsQ, err := sanitize.SQLQuery( fmt.Sprintf("SELECT nspname FROM pg_catalog.pg_namespace WHERE nspname %s $1 ESCAPE '!' AND nspname NOT LIKE 'pg!_%%' ESCAPE '!' AND nspname != 'information_schema'", op), likePattern, @@ -131,10 +109,9 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (v return schemas, hidden, nil } -// resolveIncludedSchemas resolves config.DBSchemaInclude against conn, -// applies config.DBSchemaExclude filtering, and logs schema-set drift -// against the previous resolution cached on config (see -// Config.previouslyResolvedSchemas) +// resolveIncludedSchemas resolves cfg.DBSchemaInclude against conn, applies +// cfg.DBSchemaExclude filtering, and warns about schema-set drift against +// the previously resolved set cached on cfg. func resolveIncludedSchemas(ctx context.Context, conn *pgconn.PgConn, cfg *Config) ([]string, error) { schemas, inaccessibleSchemas, err := resolveSchemas(ctx, conn, cfg.DBSchemaInclude) if err != nil { @@ -199,17 +176,9 @@ func resolveIncludedSchemas(ctx context.Context, conn *pgconn.PgConn, cfg *Confi } // resolveExistingTables returns the quoted names of the publishable base -// tables (including partitioned tables) that exist in the given (already -// quoted) schema. -// -// Used to resolve a schema glob × table list combination per-schema, since a -// matched schema may not contain a configured table (e.g. still being -// provisioned) or may have a same-named view/foreign/temporary table -// instead. table_type is restricted to 'BASE TABLE' so either case is -// treated as missing and falls through to the caller's "not found, skipping" -// warning - otherwise CreatePublication's FOR TABLE clause would reference -// an unpublishable relation and fail setup for every matched schema, not -// just the drifted one. +// tables in the given (already quoted) schema. Restricted to table_type = +// 'BASE TABLE' so a same-named view or foreign table is treated as missing +// rather than breaking CreatePublication. func resolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchema string) (map[string]struct{}, error) { schema, err := sanitize.UnquotePostgresIdentifier(quotedSchema) if err != nil { @@ -286,23 +255,11 @@ func escapeLike(s string) string { return b.String() } -// schemaMatchesExcludePattern reports whether quotedSchemaName - a schema -// identifier in the same quoted form resolveSchemas returns - matches -// excludePattern, a schema_exclude entry written in the same shape as a -// schema_include value (an exact name, a '*' glob, or a double-quoted exact -// identifier). -// -// Matching happens entirely in memory against a schema list we've already -// resolved, unlike resolveSchemas which queries the database: schema_exclude -// only ever narrows a candidate set that's already been fetched, so there's -// no reason to pay for another round-trip per exclude pattern. Semantics -// mirror schemaPatternToLike without going through SQL: a quoted pattern is -// an exact, case-sensitive match on the unquoted name; an unquoted pattern is -// matched case-insensitively with '*' as a wildcard. -// -// Returns an error only when a quoted operand fails to unquote. This should -// only happen for a malformed excludePattern in practice, since -// quotedSchemaName is always freshly quoted by resolveSchemas. +// schemaMatchesExcludePattern reports whether quotedSchemaName matches +// excludePattern, using the same pattern syntax as schema_include (exact +// name, '*' glob, or quoted exact identifier). Matches in memory against an +// already-resolved schema list, so no extra DB round-trip is needed. Returns +// an error only if a quoted operand fails to unquote. func schemaMatchesExcludePattern(quotedSchemaName, excludePattern string) (bool, error) { schemaName, err := sanitize.UnquotePostgresIdentifier(quotedSchemaName) if err != nil { @@ -324,13 +281,9 @@ func schemaMatchesExcludePattern(quotedSchemaName, excludePattern string) (bool, return re.MatchString(strings.ToLower(schemaName)), nil } -// diffSchemaSets reports which schemas are present in current but not -// previous (added) and vice versa (removed), used to detect schema-set drift -// between the schema_include resolution done on this connect and the one -// done on the previous connect/reconnect. A nil previous (the very first -// resolution in the process's lifetime) is not a meaningful "everything was -// just added" drift, so callers should only act on the result when previous -// is non-nil. +// diffSchemaSets reports schemas present in current but not previous +// (added) and vice versa (removed). Callers should ignore the result when +// previous is nil - that's the first resolution, not real drift. func diffSchemaSets(previous, current []string) (added, removed []string) { previousSet := make(map[string]struct{}, len(previous)) for _, schema := range previous { @@ -353,10 +306,8 @@ func diffSchemaSets(previous, current []string) (added, removed []string) { return added, removed } -// globToRegexp compiles an unquoted glob pattern (using '*' as a wildcard) -// into an anchored regexp - the in-memory equivalent of globToLike for -// callers matching against values already held in Go rather than via a SQL -// LIKE clause. +// globToRegexp compiles an unquoted glob pattern ('*' as wildcard) into an +// anchored regexp - the in-memory equivalent of globToLike. func globToRegexp(pattern string) (*regexp.Regexp, error) { parts := strings.Split(pattern, "*") for i, part := range parts { diff --git a/internal/plugins/cdctest/cdc_conformance_test.go b/internal/plugins/cdctest/cdc_conformance_test.go index 6a3980caba..cd5eac4de5 100644 --- a/internal/plugins/cdctest/cdc_conformance_test.go +++ b/internal/plugins/cdctest/cdc_conformance_test.go @@ -54,13 +54,6 @@ var canonicalFields = []string{ "stream_snapshot", } -// conditionalConnectors lists CDC inputs that are only registered under specific -// build tags (e.g. cgo). They are exempt from the stale-entry guard when not -// registered, but are still subject to conformance checks when they are. -var conditionalConnectors = map[string]bool{ - "tigerbeetle_cdc": true, // requires cgo -} - // knownNonConformant waives specific (connector → field → reason) checks that // have not yet been migrated. New connectors default to strict. Populated from // the actual registry state; shrink it as connectors converge. @@ -100,11 +93,11 @@ var knownNonConformant = map[string]map[string]string{ "max_parallel_snapshot_tables": "uses max_parallel_snapshot_objects; migrate", }, "tigerbeetle_cdc": { - "checkpoint_cache": "uses progress_cache; migrate to checkpoint_cache", - "checkpoint_limit": "no discrete checkpoint limit; TigerBeetle CDC is pure streaming", - "snapshot_max_batch_size": "no snapshot phase; TigerBeetle CDC is pure streaming with no initial snapshot", - "max_parallel_snapshot_tables": "no snapshot phase; TigerBeetle CDC is pure streaming with no initial snapshot", - "stream_snapshot": "no snapshot phase; TigerBeetle CDC is pure streaming with no initial snapshot", + "checkpoint_cache": "non-relational; §5 applicability under triage", + "checkpoint_limit": "non-relational; §5 applicability under triage", + "snapshot_max_batch_size": "non-relational; §5 applicability under triage", + "max_parallel_snapshot_tables": "non-relational; §5 applicability under triage", + "stream_snapshot": "non-relational; §5 applicability under triage", }, } @@ -186,10 +179,6 @@ func TestCDCConformance(t *testing.T) { } for name := range knownNonConformant { if _, ok := registered[name]; !ok { - if conditionalConnectors[name] { - t.Logf("SKIPPED stale check for %q: not registered in this build (conditional build tag)", name) - continue - } t.Errorf("stale knownNonConformant entry %q is not a registered CDC input; remove it", name) } } From 591d750493701b4b6ac9246459afcd3dc534e3ff Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Fri, 21 Aug 2026 10:41:32 +0100 Subject: [PATCH 46/48] postgres_cdc: improve schema discovery on reconnect loops --- .../pglogicalstream/logical_stream.go | 19 ++++----- .../pglogicalstream/schema_resolver.go | 42 +++++++++++++------ .../schema_resolver_integration_test.go | 2 +- .../pglogicalstream/schema_resolver_test.go | 2 +- 4 files changed, 40 insertions(+), 25 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index ddeaf31a9c..e557d57afa 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -120,21 +120,20 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { normalizedTables = append(normalizedTables, normalized) } - // With no explicit table list, auto-discover every base table in each - // matched (and schema_exclude-filtered) schema and publish them - // explicitly. Without this, an empty tables list would fall through to - // CreatePublication's FOR ALL TABLES fallback below, which replicates - // every schema in the database and silently defeats both schema_include - // and schema_exclude. + // tables empty here would otherwise fall through to CreatePublication's + // FOR ALL TABLES fallback, replicating the whole database and defeating + // schema_include/schema_exclude - auto-discover per matched schema instead. autoDiscoverTables := len(normalizedTables) == 0 + existingTablesBySchema, err := resolveExistingTables(ctx, dbConn, schemas) + if err != nil { + return nil, fmt.Errorf("resolving tables in schema(s) %v: %w", schemas, err) + } + tables = make([]TableFQN, 0, len(schemas)*len(normalizedTables)) foundTables := make(map[string]bool, len(normalizedTables)) for _, schema := range schemas { - existingTables, err := resolveExistingTables(ctx, dbConn, schema) - if err != nil { - return nil, fmt.Errorf("resolving tables in schema %q: %w", schema, err) - } + existingTables := existingTablesBySchema[schema] if autoDiscoverTables { for table := range existingTables { tables = append(tables, TableFQN{Schema: schema, Table: table}) diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index eef25e3c89..0c2f3a637a 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -176,32 +176,48 @@ func resolveIncludedSchemas(ctx context.Context, conn *pgconn.PgConn, cfg *Confi } // resolveExistingTables returns the quoted names of the publishable base -// tables in the given (already quoted) schema. Restricted to table_type = -// 'BASE TABLE' so a same-named view or foreign table is treated as missing -// rather than breaking CreatePublication. -func resolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchema string) (map[string]struct{}, error) { - schema, err := sanitize.UnquotePostgresIdentifier(quotedSchema) - if err != nil { - return nil, fmt.Errorf("unquoting schema identifier %q: %w", quotedSchema, err) +// tables in each of the given (already quoted) schemas, keyed by quoted +// schema name. A single query covering every schema, rather than one per +// schema, keeps this to one round-trip regardless of tenant count - the +// difference between one query and, say, one hundred on a multi-tenant, +// schema-per-tenant database on every connect and reconnect. Restricted to +// table_type = 'BASE TABLE' so a same-named view or foreign table is treated +// as missing rather than breaking CreatePublication. +func resolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchemas []string) (map[string]map[string]struct{}, error) { + rawToQuoted := make(map[string]string, len(quotedSchemas)) + args := make([]any, len(quotedSchemas)) + placeholders := make([]string, len(quotedSchemas)) + for i, quotedSchema := range quotedSchemas { + schema, err := sanitize.UnquotePostgresIdentifier(quotedSchema) + if err != nil { + return nil, fmt.Errorf("unquoting schema identifier %q: %w", quotedSchema, err) + } + rawToQuoted[schema] = quotedSchema + args[i] = schema + placeholders[i] = fmt.Sprintf("$%d", i+1) } q, err := sanitize.SQLQuery( - "SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type = 'BASE TABLE'", - schema, + fmt.Sprintf("SELECT table_schema, table_name FROM information_schema.tables WHERE table_schema IN (%s) AND table_type = 'BASE TABLE'", strings.Join(placeholders, ", ")), + args..., ) if err != nil { - return nil, fmt.Errorf("building table resolution query for schema %q: %w", quotedSchema, err) + return nil, fmt.Errorf("building table resolution query for schema(s) %v: %w", quotedSchemas, err) } results, err := conn.Exec(ctx, q).ReadAll() if err != nil { - return nil, fmt.Errorf("querying tables in schema %q: %w", quotedSchema, err) + return nil, fmt.Errorf("querying tables in schema(s) %v: %w", quotedSchemas, err) } - existing := map[string]struct{}{} + existing := make(map[string]map[string]struct{}, len(quotedSchemas)) + for _, quotedSchema := range quotedSchemas { + existing[quotedSchema] = map[string]struct{}{} + } if len(results) > 0 { for _, row := range results[0].Rows { - existing[sanitize.QuotePostgresIdentifier(string(row[0]))] = struct{}{} + quotedSchema := rawToQuoted[string(row[0])] + existing[quotedSchema][sanitize.QuotePostgresIdentifier(string(row[1]))] = struct{}{} } } return existing, nil diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go b/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go index 91243a6802..a7afbfa48f 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 Redpanda Data, Inc. +// Copyright 2026 Redpanda Data, Inc. // // Licensed as a Redpanda Enterprise file under the Redpanda Community // License (the "License"); you may not use this file except in compliance with diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go b/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go index ae1ee89a92..7931fddb81 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 Redpanda Data, Inc. +// Copyright 2026 Redpanda Data, Inc. // // Licensed as a Redpanda Enterprise file under the Redpanda Community // License (the "License"); you may not use this file except in compliance with From 0a86a33da4cfa1f320e6d5544391b7a3959aad03 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Fri, 21 Aug 2026 11:10:52 +0100 Subject: [PATCH 47/48] clean up logging --- internal/impl/postgresql/pglogicalstream/logical_stream.go | 2 +- internal/impl/postgresql/pglogicalstream/schema_resolver.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index e557d57afa..cb48c712bf 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -153,7 +153,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { if len(tables) == 0 { return nil, fmt.Errorf("no tables found in schema(s) %v matching schema_include pattern %q", schemas, config.DBSchemaInclude) } - config.Logger.Infof("%q has no `tables` list configured: auto-discovered %d table(s) across %d schema(s)", config.DBSchemaInclude, len(tables), len(schemas)) + config.Logger.Debugf("%q has no `tables` list configured: auto-discovered %d table(s) across %d schema(s)", config.DBSchemaInclude, len(tables), len(schemas)) } else { // A table must exist in at least one matched schema. Missing from some // (but not all) matched schemas is tolerated above as a multi-tenant gap; diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 0c2f3a637a..2304e0477f 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -143,7 +143,7 @@ func resolveIncludedSchemas(ctx context.Context, conn *pgconn.PgConn, cfg *Confi remaining = append(remaining, schema) } if len(excluded) > 0 { - cfg.Logger.Infof("schema_exclude %v excluded %d schema(s) %v from schema_include pattern %q; %d schema(s) remain: %v", cfg.DBSchemaExclude, len(excluded), excluded, cfg.DBSchemaInclude, len(remaining), remaining) + cfg.Logger.Debugf("schema_exclude %v excluded %d schema(s) %v from schema_include pattern %q; %d schema(s) remain: %v", cfg.DBSchemaExclude, len(excluded), excluded, cfg.DBSchemaInclude, len(remaining), remaining) } schemas = remaining } @@ -159,7 +159,7 @@ func resolveIncludedSchemas(ctx context.Context, conn *pgconn.PgConn, cfg *Confi } return nil, fmt.Errorf("no schemas found matching schema_include pattern %q", cfg.DBSchemaInclude) } - cfg.Logger.Infof("schema_include pattern %q resolved to %d schema(s): %v", cfg.DBSchemaInclude, len(schemas), schemas) + cfg.Logger.Debugf("schema_include pattern %q resolved to %d schema(s): %v", cfg.DBSchemaInclude, len(schemas), schemas) if cfg.previouslyResolvedSchemas != nil { added, removed := diffSchemaSets(cfg.previouslyResolvedSchemas, schemas) From bd0346b610cbc5cce5e0318be6b669b5af03d501 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Sat, 22 Aug 2026 00:09:34 +0100 Subject: [PATCH 48/48] move to package --- internal/impl/postgresql/input_pg_stream.go | 9 +- .../impl/postgresql/pglogicalstream/config.go | 10 +- .../pglogicalstream/logical_stream.go | 15 +- .../resolver.go} | 173 ++++++++++-------- .../resolver_integration_test.go} | 53 +++++- .../resolver_test.go} | 2 +- 6 files changed, 165 insertions(+), 97 deletions(-) rename internal/impl/postgresql/pglogicalstream/{schema_resolver.go => multischema/resolver.go} (76%) rename internal/impl/postgresql/pglogicalstream/{schema_resolver_integration_test.go => multischema/resolver_integration_test.go} (76%) rename internal/impl/postgresql/pglogicalstream/{schema_resolver_test.go => multischema/resolver_test.go} (99%) diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index becbe1f522..84b8d2044c 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -27,6 +27,7 @@ import ( "github.com/redpanda-data/connect/v4/internal/asyncroutine" "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream" + "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/multischema" "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/sanitize" "github.com/redpanda-data/connect/v4/internal/license" ) @@ -491,14 +492,18 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser snapshotMetrics := mgr.Metrics().NewGauge("postgres_snapshot_progress", "table") replicationLag := mgr.Metrics().NewGauge("postgres_replication_lag_bytes") + var schemaResolver *multischema.Resolver + if schemaInclude != "" { + schemaResolver = multischema.NewResolver(schemaInclude, schemaExclude) + } + i := &pgStreamInput{ streamConfig: &pglogicalstream.Config{ DBConfig: pgConnConfig, TLSConfig: pgConnConfig.TLSConfig, DBRawDSN: dsn, DBSchema: schema, - DBSchemaInclude: schemaInclude, - DBSchemaExclude: schemaExclude, + SchemaResolver: schemaResolver, DBTables: tables, RefreshAuthToken: iamAuthTokenBuilder, diff --git a/internal/impl/postgresql/pglogicalstream/config.go b/internal/impl/postgresql/pglogicalstream/config.go index d8400b7c2f..3c87d0bcaa 100644 --- a/internal/impl/postgresql/pglogicalstream/config.go +++ b/internal/impl/postgresql/pglogicalstream/config.go @@ -16,6 +16,8 @@ import ( "github.com/jackc/pgx/v5/pgconn" "github.com/redpanda-data/benthos/v4/public/service" + + "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/multischema" ) // Config is the configuration for the pglogicalstream plugin @@ -27,11 +29,9 @@ type Config struct { DBSchema string DBTables []string - // Multi schema support - DBSchemaInclude string - DBSchemaExclude []string - previouslyResolvedSchemas []string // track and report on resolved schemas between reconnects - previouslyInaccessibleSchemas []string + // SchemaResolver resolves schema_include/schema_exclude into the schemas + // to replicate. Non-nil only when schema_include is set. + SchemaResolver *multischema.Resolver // Refreshes short lived IAM auth token that is treated as a password RefreshAuthToken func(ctx context.Context) error diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index cb48c712bf..9b29cfdb95 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -26,6 +26,7 @@ import ( "github.com/redpanda-data/benthos/v4/public/service" "github.com/redpanda-data/connect/v4/internal/asyncroutine" + "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/multischema" "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/sanitize" ) @@ -102,11 +103,11 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { tables []TableFQN schema string ) - if config.DBSchemaInclude != "" { + if config.SchemaResolver != nil { if config.SignalTableName != "" { return nil, errors.New("signal_table_name is not supported when schema_include is set") } - schemas, err := resolveIncludedSchemas(ctx, dbConn, config) + schemas, err := config.SchemaResolver.Resolve(ctx, dbConn, config.Logger) if err != nil { return nil, err } @@ -125,7 +126,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { // schema_include/schema_exclude - auto-discover per matched schema instead. autoDiscoverTables := len(normalizedTables) == 0 - existingTablesBySchema, err := resolveExistingTables(ctx, dbConn, schemas) + existingTablesBySchema, err := multischema.ResolveExistingTables(ctx, dbConn, schemas) if err != nil { return nil, fmt.Errorf("resolving tables in schema(s) %v: %w", schemas, err) } @@ -142,7 +143,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } for _, table := range normalizedTables { if _, ok := existingTables[table]; !ok { - config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched schema_include pattern %q but does not contain this table)", schema, table, schema, config.DBSchemaInclude) + config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched schema_include pattern %q but does not contain this table)", schema, table, schema, config.SchemaResolver.Include) continue } tables = append(tables, TableFQN{Schema: schema, Table: table}) @@ -151,9 +152,9 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } if autoDiscoverTables { if len(tables) == 0 { - return nil, fmt.Errorf("no tables found in schema(s) %v matching schema_include pattern %q", schemas, config.DBSchemaInclude) + return nil, fmt.Errorf("no tables found in schema(s) %v matching schema_include pattern %q", schemas, config.SchemaResolver.Include) } - config.Logger.Debugf("%q has no `tables` list configured: auto-discovered %d table(s) across %d schema(s)", config.DBSchemaInclude, len(tables), len(schemas)) + config.Logger.Debugf("%q has no `tables` list configured: auto-discovered %d table(s) across %d schema(s)", config.SchemaResolver.Include, len(tables), len(schemas)) } else { // A table must exist in at least one matched schema. Missing from some // (but not all) matched schemas is tolerated above as a multi-tenant gap; @@ -166,7 +167,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } } if len(missingTables) > 0 { - return nil, fmt.Errorf("table(s) %v not found in any schema matching schema_include pattern %q", missingTables, config.DBSchemaInclude) + return nil, fmt.Errorf("table(s) %v not found in any schema matching schema_include pattern %q", missingTables, config.SchemaResolver.Include) } } } else { diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver.go similarity index 76% rename from internal/impl/postgresql/pglogicalstream/schema_resolver.go rename to internal/impl/postgresql/pglogicalstream/multischema/resolver.go index 2304e0477f..4579d90eb2 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver.go @@ -6,7 +6,10 @@ // // https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md -package pglogicalstream +// Package multischema resolves a schema_include/schema_exclude +// configuration (replicating from multiple PostgreSQL schemas matched by a +// glob pattern) into the concrete set of schemas and tables to replicate. +package multischema import ( "context" @@ -17,14 +20,95 @@ import ( "github.com/jackc/pgx/v5/pgconn" + "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/sanitize" ) -// schemaPatternToLike converts a schema name or glob pattern into the LIKE -// pattern used by resolveSchemas, plus whether it must be matched -// case-sensitively. Quoted identifiers are matched exactly and -// case-sensitively; unquoted patterns use '*' as a wildcard and match -// case-insensitively, mirroring how PostgreSQL folds unquoted identifiers. +type Resolver struct { + // Include is the schema_include glob pattern. + Include string + // Exclude is the schema_exclude list, evaluated against the schemas + // matched by Include. + Exclude []string + + previouslyResolved []string + previouslyInaccessible []string +} + +// NewResolver returns a Resolver for the given schema_include/schema_exclude +// configuration. Callers should only construct one when schema_include is +// set. +func NewResolver(include string, exclude []string) *Resolver { + return &Resolver{Include: include, Exclude: exclude} +} + +// Resolve resolves r.Include against conn, applies r.Exclude filtering, and +// warns about schema-set drift against the previously resolved set from an +// earlier call to Resolve on this same Resolver. +func (r *Resolver) Resolve(ctx context.Context, conn *pgconn.PgConn, logger *service.Logger) ([]string, error) { + schemas, inaccessibleSchemas, err := resolveSchemas(ctx, conn, r.Include) + if err != nil { + return nil, fmt.Errorf("resolving schema_include pattern %q: %w", r.Include, err) + } + matchedSchemas := schemas + + if len(r.Exclude) > 0 { + // Filtering happens entirely against the schemas slice we already + // fetched above - no extra DB round-trips per exclude pattern. + var excluded []string + remaining := make([]string, 0, len(schemas)) + for _, schema := range schemas { + var isExcluded bool + for _, pattern := range r.Exclude { + matched, err := schemaMatchesExcludePattern(schema, pattern) + if err != nil { + return nil, fmt.Errorf("evaluating schema_exclude pattern %q against schema %q: %w", pattern, schema, err) + } + if matched { + isExcluded = true + break + } + } + if isExcluded { + excluded = append(excluded, schema) + continue + } + remaining = append(remaining, schema) + } + if len(excluded) > 0 { + logger.Debugf("schema_exclude %v excluded %d schema(s) %v from schema_include pattern %q; %d schema(s) remain: %v", r.Exclude, len(excluded), excluded, r.Include, len(remaining), remaining) + } + schemas = remaining + } + + if len(inaccessibleSchemas) > 0 && !slices.Equal(inaccessibleSchemas, r.previouslyInaccessible) { + logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", r.Include, inaccessibleSchemas) + } + r.previouslyInaccessible = slices.Clone(inaccessibleSchemas) + + if len(schemas) == 0 { + if len(matchedSchemas) > 0 { + return nil, fmt.Errorf("schema_include pattern %q matched schema(s) %v, but schema_exclude %v excluded all of them", r.Include, matchedSchemas, r.Exclude) + } + return nil, fmt.Errorf("no schemas found matching schema_include pattern %q", r.Include) + } + logger.Debugf("schema_include pattern %q resolved to %d schema(s): %v", r.Include, len(schemas), schemas) + + if r.previouslyResolved != nil { + added, removed := diffSchemaSets(r.previouslyResolved, schemas) + if len(added) > 0 { + logger.Warnf("schema_include pattern %q now also matches schema(s) %v that did not match on the previous connect; their tables are being added to the publication, but any rows already in them will NOT be snapshotted even if stream_snapshot is enabled - only changes made from now on will be captured", r.Include, added) + } + if len(removed) > 0 { + logger.Warnf("schema(s) %v no longer match schema_include pattern %q (dropped, renamed, or the role lost USAGE) since the previous connect; their tables are being removed from the publication and will stop replicating", removed, r.Include) + } + } + r.previouslyResolved = slices.Clone(schemas) + + return schemas, nil +} + func schemaPatternToLike(pattern string) (likePattern string, caseSensitive bool, err error) { if strings.HasPrefix(pattern, `"`) { unquoted, err := sanitize.UnquotePostgresIdentifier(pattern) @@ -36,13 +120,6 @@ func schemaPatternToLike(pattern string) (likePattern string, caseSensitive bool return globToLike(strings.ToLower(pattern)), false, nil } -// resolveSchemas returns the schemas matching pattern that the role can see -// (visibleSchemas), plus schemas that also match but are hidden by missing -// privileges (inaccessibleSchemas) - surfaced separately so callers can warn -// instead of silently dropping them. System schemas (pg_* and -// information_schema) are always excluded. Returned names are quoted -// PostgreSQL identifiers. A nil visibleSchemas with a nil error means no -// match - callers should treat that as an error. func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (visibleSchemas, inaccessibleSchemas []string, err error) { likePattern, caseSensitive, err := schemaPatternToLike(pattern) if err != nil { @@ -109,73 +186,7 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (v return schemas, hidden, nil } -// resolveIncludedSchemas resolves cfg.DBSchemaInclude against conn, applies -// cfg.DBSchemaExclude filtering, and warns about schema-set drift against -// the previously resolved set cached on cfg. -func resolveIncludedSchemas(ctx context.Context, conn *pgconn.PgConn, cfg *Config) ([]string, error) { - schemas, inaccessibleSchemas, err := resolveSchemas(ctx, conn, cfg.DBSchemaInclude) - if err != nil { - return nil, fmt.Errorf("resolving schema_include pattern %q: %w", cfg.DBSchemaInclude, err) - } - matchedSchemas := schemas - - if len(cfg.DBSchemaExclude) > 0 { - // Filtering happens entirely against the schemas slice we already - // fetched above - no extra DB round-trips per exclude pattern. - var excluded []string - remaining := make([]string, 0, len(schemas)) - for _, schema := range schemas { - var isExcluded bool - for _, pattern := range cfg.DBSchemaExclude { - matched, err := schemaMatchesExcludePattern(schema, pattern) - if err != nil { - return nil, fmt.Errorf("evaluating schema_exclude pattern %q against schema %q: %w", pattern, schema, err) - } - if matched { - isExcluded = true - break - } - } - if isExcluded { - excluded = append(excluded, schema) - continue - } - remaining = append(remaining, schema) - } - if len(excluded) > 0 { - cfg.Logger.Debugf("schema_exclude %v excluded %d schema(s) %v from schema_include pattern %q; %d schema(s) remain: %v", cfg.DBSchemaExclude, len(excluded), excluded, cfg.DBSchemaInclude, len(remaining), remaining) - } - schemas = remaining - } - - if len(inaccessibleSchemas) > 0 && !slices.Equal(inaccessibleSchemas, cfg.previouslyInaccessibleSchemas) { - cfg.Logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", cfg.DBSchemaInclude, inaccessibleSchemas) - } - cfg.previouslyInaccessibleSchemas = slices.Clone(inaccessibleSchemas) - - if len(schemas) == 0 { - if len(matchedSchemas) > 0 { - return nil, fmt.Errorf("schema_include pattern %q matched schema(s) %v, but schema_exclude %v excluded all of them", cfg.DBSchemaInclude, matchedSchemas, cfg.DBSchemaExclude) - } - return nil, fmt.Errorf("no schemas found matching schema_include pattern %q", cfg.DBSchemaInclude) - } - cfg.Logger.Debugf("schema_include pattern %q resolved to %d schema(s): %v", cfg.DBSchemaInclude, len(schemas), schemas) - - if cfg.previouslyResolvedSchemas != nil { - added, removed := diffSchemaSets(cfg.previouslyResolvedSchemas, schemas) - if len(added) > 0 { - cfg.Logger.Warnf("schema_include pattern %q now also matches schema(s) %v that did not match on the previous connect; their tables are being added to the publication, but any rows already in them will NOT be snapshotted even if stream_snapshot is enabled - only changes made from now on will be captured", cfg.DBSchemaInclude, added) - } - if len(removed) > 0 { - cfg.Logger.Warnf("schema(s) %v no longer match schema_include pattern %q (dropped, renamed, or the role lost USAGE) since the previous connect; their tables are being removed from the publication and will stop replicating", removed, cfg.DBSchemaInclude) - } - } - cfg.previouslyResolvedSchemas = slices.Clone(schemas) - - return schemas, nil -} - -// resolveExistingTables returns the quoted names of the publishable base +// ResolveExistingTables returns the quoted names of the publishable base // tables in each of the given (already quoted) schemas, keyed by quoted // schema name. A single query covering every schema, rather than one per // schema, keeps this to one round-trip regardless of tenant count - the @@ -183,7 +194,7 @@ func resolveIncludedSchemas(ctx context.Context, conn *pgconn.PgConn, cfg *Confi // schema-per-tenant database on every connect and reconnect. Restricted to // table_type = 'BASE TABLE' so a same-named view or foreign table is treated // as missing rather than breaking CreatePublication. -func resolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchemas []string) (map[string]map[string]struct{}, error) { +func ResolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchemas []string) (map[string]map[string]struct{}, error) { rawToQuoted := make(map[string]string, len(quotedSchemas)) args := make([]any, len(quotedSchemas)) placeholders := make([]string, len(quotedSchemas)) diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go similarity index 76% rename from internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go rename to internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go index a7afbfa48f..a2497e08b4 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go @@ -6,11 +6,12 @@ // // https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md -package pglogicalstream +package multischema import ( "context" "database/sql" + "fmt" "testing" "time" @@ -19,10 +20,60 @@ import ( "github.com/jackc/pgx/v5/pgconn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" "github.com/redpanda-data/benthos/v4/public/service/integration" ) +// closeConn and createDockerInstance mirror the identically named helpers in +// pglogicalstream/pglogrepl_test.go - duplicated here rather than shared, +// since they're small, self-contained, and this package must not import the +// pglogicalstream test package. +func closeConn(t testing.TB, conn *pgconn.PgConn) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + require.NoError(t, conn.Close(ctx)) +} + +func createDockerInstance(t *testing.T) (cleanup func(), dbURL string) { + ctr, err := testcontainers.Run(t.Context(), "postgres:16", + testcontainers.WithExposedPorts("5432/tcp"), + testcontainers.WithEnv(map[string]string{ + "POSTGRES_PASSWORD": "secret", + "POSTGRES_USER": "user_name", + "POSTGRES_DB": "dbname", + }), + testcontainers.WithCmd("postgres", "-c", "wal_level=logical"), + testcontainers.WithWaitStrategy( + wait.ForListeningPort("5432/tcp").WithStartupTimeout(2*time.Minute), + ), + ) + testcontainers.CleanupContainer(t, ctr) + require.NoError(t, err) + + host, err := ctr.Host(t.Context()) + require.NoError(t, err) + mp, err := ctr.MappedPort(t.Context(), "5432/tcp") + require.NoError(t, err) + + databaseURL := fmt.Sprintf("user=user_name password=secret dbname=dbname sslmode=disable host=%s port=%s replication=database", host, mp.Port()) + + var db *sql.DB + require.Eventually(t, func() bool { + if db, err = sql.Open("postgres", databaseURL); err != nil { + return false + } + return db.Ping() == nil + }, 2*time.Minute, time.Second) + + cleanup = func() { + // Container cleanup is handled by testcontainers.CleanupContainer + } + + return cleanup, databaseURL +} + // TestIntegrationResolveSchemasReportsInaccessibleSchemas verifies that a // schema pattern matching a schema the connecting role lacks USAGE on is // reported via inaccessibleSchemas rather than silently dropped, since diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver_test.go similarity index 99% rename from internal/impl/postgresql/pglogicalstream/schema_resolver_test.go rename to internal/impl/postgresql/pglogicalstream/multischema/resolver_test.go index 7931fddb81..d06b5362dd 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver_test.go @@ -6,7 +6,7 @@ // // https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md -package pglogicalstream +package multischema import ( "testing"