diff --git a/docs/_src/content.py b/docs/_src/content.py index 05b8c682..138902ad 100644 --- a/docs/_src/content.py +++ b/docs/_src/content.py @@ -324,7 +324,7 @@ def page(group, slug, title, summary, *sections, source="src/Migrator/Framework/ ''', ''' migration.Delete.Index("IX_Users_Name").FromTable("Users"); ''')), - section("Provider options", '

Index definitions also expose IncludeColumns, FilterItems and Clustered. These options are provider-specific. Oracle rejects included and clustered index requests; SQLite reconstruction rejects existing index SQL with explicit COLLATE clauses. Preview handles simple indexes and rejects unsupported options.

'), + section("Provider options", '

Index definitions also expose IncludeColumns, FilterItems and Clustered. SQL Server (2008+), PostgreSQL and SQLite support filters on any table column, including columns outside KeyColumns. EqualTo or NotEqualTo with null (or DBNull.Value) becomes IS NULL or IS NOT NULL. GetIndexes(table), also available as Schema.Table(table).Indexes() in fluent migrations, reads back the keys, included columns, flags and supported FilterItems. Filters preserve null checks and escaped string values.

UnsupportedFilterBehavior defaults to UnsupportedIndexFilterBehavior.Throw. Set it to Ignore, or append OnUnsupportedFilter(UnsupportedIndexFilterBehavior.Ignore) in fluent code, to create an unfiltered index on a provider that cannot apply the filters. For unique indexes this enforces uniqueness across all rows. This option only affects unsupported filters; it does not suppress other invalid options or execution failures.

Oracle retains its limited non-unique, key-column expression emulation and cannot read those expressions back as FilterItems; non-key filters and unique filtered indexes use the chosen unsupported behavior. Oracle rejects included and clustered index requests; SQLite reconstruction rejects existing index SQL with explicit COLLATE clauses.

The fallback policy is an authoring option and is not stored in database metadata. Preview handles simple indexes and rejects filtered indexes even in Ignore mode.

'), section("Unique index or unique constraint?", '

Use UniqueConstraint for a table-level invariant and an Index with Unique for an index definition. Do not infer ownership from a generated name. SQLite RemoveAllIndexes preserves declared table UNIQUE constraints; remove those through the constraint APIs. Check query plans and data cardinality when choosing index keys.

'), source="src/Migrator/Framework/Index.cs") page("Schema basics", "constraints", "Keys and constraints", "Declare table invariants independently of column attributes.", diff --git a/docs/assets/search-index.json b/docs/assets/search-index.json index 54f54c38..e901ce84 100644 --- a/docs/assets/search-index.json +++ b/docs/assets/search-index.json @@ -81,7 +81,7 @@ "group": "Schema basics", "summary": "An index is a separate schema object, even when it enforces uniqueness.", "url": "guide/indexes.html", - "text": " Use an explicit name so the index can be inspected or removed later. Fluent Create.Index(name).OnTable(table).WithColumns(...) names each part explicitly and preserves column order. Append Unique(), Clustered(), IncludeColumns(...) or WithFilter(...). For an existing Index definition, use Create.Index(definition).OnTable(table); fully qualify the model type if System.Index is also in scope. Index a user name Database.AddIndex(\"Users\", new DotNetProjects.Migrator.Framework.Index\n{\n Name = \"IX_Users_Name\", KeyColumns = new[] { \"Name\" }, Unique = false\n}); migration.Create.Index(\"IX_Users_Name\").OnTable(\"Users\").WithColumns(\"Name\"); Drop an index Database.RemoveIndex(\"Users\", \"IX_Users_Name\"); migration.Delete.Index(\"IX_Users_Name\").FromTable(\"Users\"); Index definitions also expose IncludeColumns, FilterItems and Clustered. These options are provider-specific. Oracle rejects included and clustered index requests; SQLite reconstruction rejects existing index SQL with explicit COLLATE clauses. Preview handles simple indexes and rejects unsupported options. Use UniqueConstraint for a table-level invariant and an Index with Unique for an index definition. Do not infer ownership from a generated name. SQLite RemoveAllIndexes preserves declared table UNIQUE constraints; remove those through the constraint APIs. Check query plans and data cardinality when choosing index keys. " + "text": " Use an explicit name so the index can be inspected or removed later. Fluent Create.Index(name).OnTable(table).WithColumns(...) names each part explicitly and preserves column order. Append Unique(), Clustered(), IncludeColumns(...) or WithFilter(...). For an existing Index definition, use Create.Index(definition).OnTable(table); fully qualify the model type if System.Index is also in scope. Index a user name Database.AddIndex(\"Users\", new DotNetProjects.Migrator.Framework.Index\n{\n Name = \"IX_Users_Name\", KeyColumns = new[] { \"Name\" }, Unique = false\n}); migration.Create.Index(\"IX_Users_Name\").OnTable(\"Users\").WithColumns(\"Name\"); Drop an index Database.RemoveIndex(\"Users\", \"IX_Users_Name\"); migration.Delete.Index(\"IX_Users_Name\").FromTable(\"Users\"); Index definitions also expose IncludeColumns, FilterItems and Clustered. SQL Server (2008+), PostgreSQL and SQLite support filters on any table column, including columns outside KeyColumns. EqualTo or NotEqualTo with null (or DBNull.Value) becomes IS NULL or IS NOT NULL. GetIndexes(table), also available as Schema.Table(table).Indexes() in fluent migrations, reads back the keys, included columns, flags and supported FilterItems. Filters preserve null checks and escaped string values. UnsupportedFilterBehavior defaults to UnsupportedIndexFilterBehavior.Throw. Set it to Ignore, or append OnUnsupportedFilter(UnsupportedIndexFilterBehavior.Ignore) in fluent code, to create an unfiltered index on a provider that cannot apply the filters. For unique indexes this enforces uniqueness across all rows. This option only affects unsupported filters; it does not suppress other invalid options or execution failures. Oracle retains its limited non-unique, key-column expression emulation and cannot read those expressions back as FilterItems; non-key filters and unique filtered indexes use the chosen unsupported behavior. Oracle rejects included and clustered index requests; SQLite reconstruction rejects existing index SQL with explicit COLLATE clauses. The fallback policy is an authoring option and is not stored in database metadata. Preview handles simple indexes and rejects filtered indexes even in Ignore mode. Use UniqueConstraint for a table-level invariant and an Index with Unique for an index definition. Do not infer ownership from a generated name. SQLite RemoveAllIndexes preserves declared table UNIQUE constraints; remove those through the constraint APIs. Check query plans and data cardinality when choosing index keys. " }, { "title": "Keys and constraints", diff --git a/docs/guide/indexes.html b/docs/guide/indexes.html index 60446bd1..05e90c39 100644 --- a/docs/guide/indexes.html +++ b/docs/guide/indexes.html @@ -16,4 +16,4 @@

Classic

Database.RemoveIndex("Users", "IX_Users_Name");

Fluent

-
migration.Delete.Index("IX_Users_Name").FromTable("Users");

Inside Up() / BuildUp(MigrationBuilder migration)

Provider options

Index definitions also expose IncludeColumns, FilterItems and Clustered. These options are provider-specific. Oracle rejects included and clustered index requests; SQLite reconstruction rejects existing index SQL with explicit COLLATE clauses. Preview handles simple indexes and rejects unsupported options.

Unique index or unique constraint?

Use UniqueConstraint for a table-level invariant and an Index with Unique for an index definition. Do not infer ownership from a generated name. SQLite RemoveAllIndexes preserves declared table UNIQUE constraints; remove those through the constraint APIs. Check query plans and data cardinality when choosing index keys.

Implementation reference ↗Improve this page ↗
+
migration.Delete.Index("IX_Users_Name").FromTable("Users");

Inside Up() / BuildUp(MigrationBuilder migration)

Provider options

Index definitions also expose IncludeColumns, FilterItems and Clustered. SQL Server (2008+), PostgreSQL and SQLite support filters on any table column, including columns outside KeyColumns. EqualTo or NotEqualTo with null (or DBNull.Value) becomes IS NULL or IS NOT NULL. GetIndexes(table), also available as Schema.Table(table).Indexes() in fluent migrations, reads back the keys, included columns, flags and supported FilterItems. Filters preserve null checks and escaped string values.

UnsupportedFilterBehavior defaults to UnsupportedIndexFilterBehavior.Throw. Set it to Ignore, or append OnUnsupportedFilter(UnsupportedIndexFilterBehavior.Ignore) in fluent code, to create an unfiltered index on a provider that cannot apply the filters. For unique indexes this enforces uniqueness across all rows. This option only affects unsupported filters; it does not suppress other invalid options or execution failures.

Oracle retains its limited non-unique, key-column expression emulation and cannot read those expressions back as FilterItems; non-key filters and unique filtered indexes use the chosen unsupported behavior. Oracle rejects included and clustered index requests; SQLite reconstruction rejects existing index SQL with explicit COLLATE clauses.

The fallback policy is an authoring option and is not stored in database metadata. Preview handles simple indexes and rejects filtered indexes even in Ignore mode.

Unique index or unique constraint?

Use UniqueConstraint for a table-level invariant and an Index with Unique for an index definition. Do not infer ownership from a generated name. SQLite RemoveAllIndexes preserves declared table UNIQUE constraints; remove those through the constraint APIs. Check query plans and data cardinality when choosing index keys.

Implementation reference ↗Improve this page ↗
diff --git a/docs/runner-guide.md b/docs/runner-guide.md index e32918c5..9eb00240 100644 --- a/docs/runner-guide.md +++ b/docs/runner-guide.md @@ -39,6 +39,36 @@ migration.Create.Index("IX_Users_Email").OnTable("Users").WithColumns("Email"); migration.Delete.Index("IX_Users_Email").FromTable("Users"); migration.Delete.Column("Email").FromTable("Users"); ``` + +Filtered indexes can use columns outside the index keys on SQL Server (2008+), PostgreSQL and SQLite. For example, enforce unique identifiers only for active users with a non-null identifier: + +```csharp +// FilterItem and FilterType are in Providers.Models.Indexes and its Enums namespace. +migration.Create.Index("UX_ActiveUsers").OnTable("Users") + .WithColumns("IpaUserIdentifier").Unique() + .WithFilter( + new FilterItem { ColumnName = "IpaUserIdentifier", Filter = FilterType.NotEqualTo, Value = null }, + new FilterItem { ColumnName = "Archive", Filter = FilterType.EqualTo, Value = 0 }) + .OnUnsupportedFilter(UnsupportedIndexFilterBehavior.Throw); +``` + +Classic migrations use the same `FilterItems` list on `Index`, with +`UnsupportedFilterBehavior = UnsupportedIndexFilterBehavior.Throw` (the default). +Choose `Ignore` to omit all filters when the provider cannot apply them. The resulting +index is unfiltered; a unique index then constrains all rows. Supported providers +still apply the filters in Ignore mode, and database errors are never swallowed. +Oracle retains its limited non-unique, key-column expression emulation; unique or +non-key filters use the unsupported behavior. Other providers without implemented +filter support, including the SQL Server 2005 dialect, also use that behavior. + +Read the definition back with `Database.GetIndexes("Users")` or +`Schema.Table("Users").Indexes()` inside a fluent migration. SQL Server, PostgreSQL +and SQLite return supported filter predicates, including null checks, together with +the index name, key order, included columns and flags. Those definitions can be used +to recreate the index. Predicates outside the `FilterItems` model (such as `OR`) fail +explicitly. Oracle expression-index metadata remains outside this read-back support. +The unsupported-filter policy is not stored in the database and reads back as the +default. SQL preview still rejects filtered indexes, including Ignore mode. Table definitions, column additions and column alterations share the same type and option methods. Each named column must specify its type. `AsDateTime()` maps diff --git a/src/Migrator.Tests/FilteredIndexTests.cs b/src/Migrator.Tests/FilteredIndexTests.cs new file mode 100644 index 00000000..2ee81485 --- /dev/null +++ b/src/Migrator.Tests/FilteredIndexTests.cs @@ -0,0 +1,349 @@ +using System; +using System.Data; +using System.Data.Common; +using System.Linq; +using DotNetProjects.Migrator; +using DotNetProjects.Migrator.Framework; +using DotNetProjects.Migrator.Framework.Fluent; +using DotNetProjects.Migrator.Providers; +using DotNetProjects.Migrator.Providers.Impl.PostgreSQL; +using DotNetProjects.Migrator.Providers.Impl.Oracle; +using DotNetProjects.Migrator.Providers.Impl.SqlServer; +using DotNetProjects.Migrator.Providers.Models.Indexes; +using DotNetProjects.Migrator.Providers.Models.Indexes.Enums; +using Microsoft.Data.Sqlite; +using NSubstitute; +using NUnit.Framework; +using Index = DotNetProjects.Migrator.Framework.Index; + +namespace Migrator.Tests; + +public class FilteredIndexTests +{ + private static Column[] Columns() => [new("IpaUserIdentifier", DbType.String, 80), new("Archive", DbType.Int32), new("select", DbType.String, 80)]; + private static Index Definition() => new() + { + Name = "UX_ActiveUsers", Unique = true, KeyColumns = ["IpaUserIdentifier"], + FilterItems = [ + new() { ColumnName = "IpaUserIdentifier", Filter = FilterType.NotEqualTo, Value = null }, + new() { ColumnName = "Archive", Filter = FilterType.EqualTo, Value = 0 }] + }; + + private sealed class SqlServerProvider(IDbConnection connection, Dialect dialect) + : SqlServerTransformationProvider(dialect, connection, "dbo", "default", null) + { + public override Column[] GetColumns(string table) => Columns(); + public override bool TableExists(string table) => true; + public override bool IndexExists(string table, string name) => false; + } + + private sealed class PostgresProvider(IDbConnection connection) + : PostgreSQLTransformationProvider(new PostgreSQLDialect(), connection, "public", "default", null) + { + public override Column[] GetColumns(string table) => Columns(); + public override bool TableExists(string table) => true; + public override bool IndexExists(string table, string name) => false; + } + + private sealed class OracleProvider(IDbConnection connection) + : OracleTransformationProvider(new OracleDialect(), connection, null, "default", null) + { + public override Column[] GetColumns(string table) => Columns(); + public override bool TableExists(string table) => true; + public override bool IndexExists(string table, string name) => false; + } + + private static IDbConnection Connection(out IDbCommand command) + { + var connection = Substitute.For(); + connection.State.Returns(ConnectionState.Open); + command = Substitute.For(); + command.CreateParameter().Returns(_ => Substitute.For()); + command.Parameters.Returns(Substitute.For()); + connection.CreateCommand().Returns(command); + return connection; + } + + [TestCase(false)] + [TestCase(true)] + public void NativeProvidersGenerateIssue199WithIncludedColumns(bool postgres) + { + var connection = Connection(out var command); + using TransformationProvider provider = postgres ? new PostgresProvider(connection) : new SqlServerProvider(connection, new SqlServerDialect()); + var index = Definition(); + index.IncludeColumns = ["select"]; + index.UnsupportedFilterBehavior = UnsupportedIndexFilterBehavior.Ignore; + var sql = provider.AddIndex("Users", index); + Assert.That(sql, Does.Contain("IS NOT NULL").And.Contain(" = 0").And.Contain("WHERE").And.Contain("INCLUDE")); + Assert.That(sql.IndexOf("INCLUDE", StringComparison.Ordinal), Is.LessThan(sql.IndexOf("WHERE", StringComparison.Ordinal))); + Assert.That(sql, Does.Contain(postgres ? "\"select\"" : "[select]")); + command.Received(1).ExecuteNonQuery(); + Assert.That(index.KeyColumns, Is.EqualTo(new[] { "IpaUserIdentifier" })); + Assert.That(index.FilterItems.Count, Is.EqualTo(2)); + } + + [TestCase(false)] + [TestCase(true)] + public void InvalidFiltersStillFailInIgnoreMode(bool missingColumn) + { + using var provider = new SqlServerProvider(Connection(out var command), new SqlServerDialect()); + var index = Definition(); + index.UnsupportedFilterBehavior = UnsupportedIndexFilterBehavior.Ignore; + if (missingColumn) index.FilterItems[0].ColumnName = "Missing"; + else index.FilterItems[0].Filter = FilterType.GreaterThan; + if (missingColumn) Assert.Throws(() => provider.AddIndex("Users", index)); + else Assert.Throws(() => provider.AddIndex("Users", index)); + command.DidNotReceive().ExecuteNonQuery(); + } + + [TestCase(ProviderTypes.Mysql)] + [TestCase(ProviderTypes.MariaDB)] + [TestCase(ProviderTypes.Firebird)] + [TestCase(ProviderTypes.IBM_DB2)] + [TestCase(ProviderTypes.IBM_Informix)] + [TestCase(ProviderTypes.Sybase)] + [TestCase(ProviderTypes.Hana)] + [TestCase(ProviderTypes.SqlServer2005)] + [TestCase(ProviderTypes.Oracle)] + public void UnsupportedFiltersThrowByDefaultAndCanBeIgnoredThroughFluent(ProviderTypes type) + { + var connection = Connection(out var command); + using var provider = type switch + { + ProviderTypes.SqlServer2005 => new SqlServerProvider(connection, new SqlServer2005Dialect()), + ProviderTypes.Oracle => new OracleProvider(connection), + _ => ProviderFactory.Create(type, connection, null) + }; + var index = Definition(); + Assert.Throws(() => provider.AddIndex("Users", index)); + command.DidNotReceive().ExecuteNonQuery(); + var builder = new MigrationBuilder(); + builder.Create.Index(index.Name).OnTable("Users").WithColumns(index.KeyColumns).Unique() + .WithFilter(index.FilterItems.ToArray()).OnUnsupportedFilter(UnsupportedIndexFilterBehavior.Ignore); + builder.Apply(provider); + command.Received(1).ExecuteNonQuery(); + Assert.That(command.CommandText, Does.Contain("UNIQUE").And.Not.Contain("WHERE")); + Assert.That(index.FilterItems.Count, Is.EqualTo(2)); + command.ClearReceivedCalls(); + index.UnsupportedFilterBehavior = UnsupportedIndexFilterBehavior.Ignore; + index.IncludeColumns = ["select"]; + if (type == ProviderTypes.SqlServer2005) return; // INCLUDE is supported by this dialect. + Assert.Throws(() => provider.AddIndex("Users", index)); + command.DidNotReceive().ExecuteNonQuery(); + } + + [Test] + public void FluentDefinitionAndBuildSnapshotsPreserveThePolicyAndNullFilters() + { + var definition = Definition(); + definition.UnsupportedFilterBehavior = UnsupportedIndexFilterBehavior.Ignore; + var builder = new MigrationBuilder(); + builder.Create.Index(definition).OnTable("Users"); + definition.UnsupportedFilterBehavior = UnsupportedIndexFilterBehavior.Throw; + definition.FilterItems.Clear(); + var built = (IndexOperation)builder.Build().Single(); + Assert.That(built.Index.UnsupportedFilterBehavior, Is.EqualTo(UnsupportedIndexFilterBehavior.Ignore)); + Assert.That(built.Index.FilterItems.Count, Is.EqualTo(2)); + var provider = Substitute.For(); + built.Apply(provider); + provider.Received().AddIndex("Users", Arg.Is(i => i.UnsupportedFilterBehavior == UnsupportedIndexFilterBehavior.Ignore && i.FilterItems[0].Value == null)); + Assert.That(((RemoveOperation)built.Reverse()).Name, Is.EqualTo(definition.Name)); + } + + [Test] + public void OracleEmulationDoesNotMutateCallerKeysOrFilters() + { + using var provider = new OracleProvider(Connection(out var command)); + var index = Definition(); + index.Unique = false; + index.KeyColumns = ["IpaUserIdentifier", "Archive"]; + var sql = provider.AddIndex("Users", index); + Assert.That(sql, Does.Contain("CASE WHEN").And.Contain("IS NOT NULL")); + Assert.That(index.KeyColumns, Is.EqualTo(new[] { "IpaUserIdentifier", "Archive" })); + Assert.That(index.FilterItems.Count, Is.EqualTo(2)); + command.Received(1).ExecuteNonQuery(); + } + + [Test] + public void FluentCanReuseCatalogDefinitionsWithNullIncludedColumns() + { + var index = Definition(); + index.IncludeColumns = null; + var builder = new MigrationBuilder(); + builder.Create.Index(index).OnTable("Users"); + var snapshot = ((IndexOperation)builder.Build().Single()).Index; + Assert.That(snapshot.IncludeColumns, Is.Empty); + Assert.That(snapshot.FilterItems.Count, Is.EqualTo(2)); + } + + [TestCase("([IpaUserIdentifier] IS NOT NULL AND [Archive]=(0))")] + [TestCase("((ipauseridentifier IS NOT NULL) AND (archive = 0))")] + [TestCase("IpaUserIdentifier IS NOT NULL AND Archive = 0")] + public void ReadsNativeCatalogNullAndNonKeyPredicates(string sql) + { + var filters = IndexFilterSql.Parse(sql, Columns()); + Assert.That(filters.Select(f => f.ColumnName), Is.EqualTo(new[] { "IpaUserIdentifier", "Archive" })); + Assert.That(filters.Select(f => f.Filter), Is.EqualTo(new[] { FilterType.NotEqualTo, FilterType.EqualTo })); + Assert.That(filters.Select(f => f.Value), Is.EqualTo(new object[] { null, 0 })); + } + + [TestCase("([select]=N'O''Brien AND (friends)' AND [Archive]=(0))")] + [TestCase("(((\"select\")::text = 'O''Brien AND (friends)'::text) AND (archive = 0))")] + [TestCase("\"select\" = 'O''Brien AND (friends)' AND Archive = 0")] + public void ReadsEscapedStringsWithoutSplittingTheirContents(string sql) + { + var filters = IndexFilterSql.Parse(sql, Columns()); + Assert.That(filters.Count, Is.EqualTo(2)); + Assert.That(filters[0].ColumnName, Is.EqualTo("select")); + Assert.That(filters[0].Value, Is.EqualTo("O'Brien AND (friends)")); + } + + [TestCase(FilterType.EqualTo, "IS NULL")] + [TestCase(FilterType.NotEqualTo, "IS NOT NULL")] + public void NullAndDbNullHaveTheSameSql(FilterType type, string expected) + { + foreach (var value in new[] { null, DBNull.Value }) + Assert.That(IndexFilterSql.Format(new SqlServerDialect(), new FilterItem { ColumnName = "select", Filter = type, Value = value }, true), Is.EqualTo("[select] " + expected)); + } + + [Test] + public void SqlServerGetIndexesReturnsAllDefinitionFieldsFromCatalog() + { + using var data = new DataTable(); + foreach (var name in new[] { "SchemaName", "TableName", "IndexName", "IndexType", "ColumnName", "FilterDefinition" }) data.Columns.Add(name); + data.Columns.Add("ColumnOrder", typeof(int)); + foreach (var name in new[] { "IsUnique", "IsPrimaryKey", "IsUniqueConstraint", "IsDescending", "IsIncludedColumn", "IsFilteredIndex" }) data.Columns.Add(name, typeof(bool)); + foreach (var (name, order, included) in new[] { ("Archive", 2, false), ("select", 3, true), ("IpaUserIdentifier", 1, false) }) + { + var row = data.NewRow(); + row["SchemaName"] = "audit"; row["TableName"] = "Users"; row["IndexName"] = "UX_ActiveUsers"; + row["IndexType"] = "NONCLUSTERED"; row["ColumnName"] = name; row["ColumnOrder"] = order; + row["FilterDefinition"] = "([IpaUserIdentifier] IS NOT NULL AND [Archive]=(0))"; + row["IsUnique"] = true; row["IsPrimaryKey"] = false; row["IsUniqueConstraint"] = false; + row["IsDescending"] = false; row["IsIncludedColumn"] = included; row["IsFilteredIndex"] = true; + data.Rows.Add(row); + } + using var provider = new SqlServerProvider(Connection(out var command), new SqlServerDialect()); + command.ExecuteReader().Returns(_ => data.CreateDataReader()); + AssertCatalogIndex(provider.GetIndexes("audit.Users").Single()); + Assert.That(command.CommandText, Does.Contain("'audit'")); + } + + [Test] + public void PostgresGetIndexesReturnsAllDefinitionFieldsFromCatalog() + { + using var data = new DataTable(); + foreach (var name in new[] { "schema_name", "table_name", "index_name", "index_definition", "index_columns", "include_columns", "partial_filter" }) data.Columns.Add(name); + foreach (var name in new[] { "is_unique", "is_clustered", "is_unique_constraint", "is_primary_constraint" }) data.Columns.Add(name, typeof(bool)); + var row = data.NewRow(); + row["schema_name"] = "audit"; row["table_name"] = "Users"; row["index_name"] = "UX_ActiveUsers"; + row["index_definition"] = "CREATE UNIQUE INDEX ..."; + row["index_columns"] = "IpaUserIdentifier, Archive"; row["include_columns"] = "select"; + row["partial_filter"] = "((ipauseridentifier IS NOT NULL) AND (archive = 0))"; + row["is_unique"] = true; row["is_clustered"] = false; row["is_unique_constraint"] = false; row["is_primary_constraint"] = false; + data.Rows.Add(row); + using var provider = new PostgresProvider(Connection(out var command)); + command.ExecuteReader().Returns(_ => data.CreateDataReader()); + AssertCatalogIndex(provider.GetIndexes("audit.Users").Single()); + Assert.That(command.CommandText, Does.Contain("to_regclass(@relation)")); + command.Parameters.Received().Add(Arg.Is(p => p.ParameterName == "relation" && p.Value.ToString().Contains("audit"))); + } + + private static void AssertCatalogIndex(Index index) + { + Assert.That(index.Name, Is.EqualTo("UX_ActiveUsers")); + Assert.That(index.KeyColumns, Is.EqualTo(new[] { "IpaUserIdentifier", "Archive" })); + Assert.That(index.IncludeColumns, Is.EqualTo(new[] { "select" })); + Assert.That(index.Unique, Is.True); + Assert.That(index.Clustered || index.PrimaryKey || index.UniqueConstraint, Is.False); + Assert.That(index.FilterItems.Select(f => f.ColumnName), Is.EqualTo(new[] { "IpaUserIdentifier", "Archive" })); + Assert.That(index.FilterItems.Select(f => f.Filter), Is.EqualTo(new[] { FilterType.NotEqualTo, FilterType.EqualTo })); + Assert.That(index.FilterItems.Select(f => f.Value), Is.EqualTo(new object[] { null, 0 })); + } + + [TestCase("Archive = 0 OR IpaUserIdentifier IS NULL")] + [TestCase("\"select\" = 'x'::text || 'y'")] + public void UnrepresentablePredicatesFailRatherThanReturnIncompleteMetadata(string sql) + => Assert.Throws(() => IndexFilterSql.Parse(sql, Columns())); + + [TestCase(false)] + [TestCase(true)] + [Category("SQLite")] + public void SQLiteReadsBackAndRebuildsTheCompleteIndex(bool fluent) + { + using var connection = new SqliteConnection("Data Source=:memory:"); + connection.Open(); + using var provider = ProviderFactory.Create(ProviderTypes.SQLite, connection, null); + VerifyRoundTrip(provider, fluent, includeColumns: false, rebuild: true); + } + + [Test, Category("SQLite")] + public void SQLiteReadsEachIndexWithEscapedStringsAndQuotedFilterColumns() + { + using var connection = new SqliteConnection("Data Source=:memory:"); + connection.Open(); + using var provider = ProviderFactory.Create(ProviderTypes.SQLite, connection, null); + provider.AddTable("FilteredUsers", Columns()); + var first = Definition(); + first.Name = "IX_Filter_Longer"; + first.FilterItems = [new() { ColumnName = "select", Filter = FilterType.EqualTo, Value = "O'Brien AND (friends)" }]; + provider.AddIndex("FilteredUsers", first); + var second = Definition(); + second.Name = "IX_Filter"; + second.FilterItems = [new() { ColumnName = "select", Filter = FilterType.EqualTo, Value = null }]; + provider.AddIndex("FilteredUsers", second); + var indexes = provider.GetIndexes("FilteredUsers"); + Assert.That(indexes.Single(i => i.Name == first.Name).FilterItems.Single().Value, Is.EqualTo("O'Brien AND (friends)")); + var nullFilter = indexes.Single(i => i.Name == second.Name).FilterItems.Single(); + Assert.That(nullFilter.ColumnName, Is.EqualTo("select")); + Assert.That(nullFilter.Filter, Is.EqualTo(FilterType.EqualTo)); + Assert.That(nullFilter.Value, Is.Null); + } + + internal static void VerifyRoundTrip(ITransformationProvider provider, bool fluent, bool includeColumns, bool rebuild = false) + { + provider.AddTable("FilteredUsers", Columns()); + var definition = Definition(); + if (includeColumns) definition.IncludeColumns = ["select"]; + if (fluent) + { + var builder = new MigrationBuilder(); + var options = builder.Create.Index(definition.Name).OnTable("FilteredUsers").WithColumns(definition.KeyColumns).Unique() + .WithFilter(definition.FilterItems.ToArray()).OnUnsupportedFilter(UnsupportedIndexFilterBehavior.Throw); + if (includeColumns) options.IncludeColumns(definition.IncludeColumns); + builder.Apply(provider); + } + else provider.AddIndex("FilteredUsers", definition); + if (rebuild) provider.ChangeColumn("FilteredUsers", new Column("select", DbType.String, 120)); + var actual = provider.GetIndexes("FilteredUsers").Single(i => i.Name.Equals(definition.Name, StringComparison.OrdinalIgnoreCase)); + Assert.That(actual.Name, Is.EqualTo(definition.Name).IgnoreCase); + Assert.That(actual.KeyColumns, Is.EqualTo(definition.KeyColumns).IgnoreCase); + Assert.That(actual.IncludeColumns ?? [], Is.EqualTo(definition.IncludeColumns).IgnoreCase); + Assert.That(actual.Unique, Is.True); + Assert.That(actual.Clustered, Is.False); + Assert.That(actual.PrimaryKey, Is.False); + Assert.That(actual.UniqueConstraint, Is.False); + Assert.That(actual.FilterItems.Select(f => f.ColumnName), Is.EquivalentTo(definition.FilterItems.Select(f => f.ColumnName)).IgnoreCase); + foreach (var filter in definition.FilterItems) + { + var read = actual.FilterItems.Single(f => f.ColumnName.Equals(filter.ColumnName, StringComparison.OrdinalIgnoreCase)); + Assert.That(read.Filter, Is.EqualTo(filter.Filter)); + Assert.That(read.Value, Is.EqualTo(filter.Value)); + } + // Recreate using metadata to prove it is usable, then verify filtered uniqueness. + provider.RemoveIndex("FilteredUsers", actual.Name); + if (fluent) + { + var builder = new MigrationBuilder(); + builder.Create.Index(actual).OnTable("FilteredUsers"); + builder.Apply(provider); + } + else provider.AddIndex("FilteredUsers", actual); + provider.Insert("FilteredUsers", ["IpaUserIdentifier", "Archive"], [null, 0]); + provider.Insert("FilteredUsers", ["IpaUserIdentifier", "Archive"], [null, 0]); + provider.Insert("FilteredUsers", ["IpaUserIdentifier", "Archive"], ["same", 1]); + provider.Insert("FilteredUsers", ["IpaUserIdentifier", "Archive"], ["same", 1]); + provider.Insert("FilteredUsers", ["IpaUserIdentifier", "Archive"], ["same", 0]); + Assert.Catch(() => provider.Insert("FilteredUsers", ["IpaUserIdentifier", "Archive"], ["same", 0])); + } +} diff --git a/src/Migrator.Tests/Providers/Generic/Generic_AddIndexTestsBase.cs b/src/Migrator.Tests/Providers/Generic/Generic_AddIndexTestsBase.cs index 1076d801..7c59e05e 100644 --- a/src/Migrator.Tests/Providers/Generic/Generic_AddIndexTestsBase.cs +++ b/src/Migrator.Tests/Providers/Generic/Generic_AddIndexTestsBase.cs @@ -1,6 +1,7 @@ using System.Data; using System.Linq; using DotNetProjects.Migrator.Framework; +using DotNetProjects.Migrator.Providers.Impl.Oracle; using DotNetProjects.Migrator.Providers.Models.Indexes; using DotNetProjects.Migrator.Providers.Models.Indexes.Enums; using Migrator.Tests.Providers.Base; @@ -55,7 +56,7 @@ public void AddIndex_IncludeColumnsContainsColumnThatExistInKeyColumns_Throws() } [Test] - public void AddIndex_ColumnNameUsedInFilterItemDoesNotExistInKeyColumns_Throws() + public void AddIndex_FilterOnNonKeyColumnIsSupportedByNativeFilteredIndexProviders() { // Arrange const string tableName = "TestTable"; @@ -68,13 +69,22 @@ public void AddIndex_ColumnNameUsedInFilterItemDoesNotExistInKeyColumns_Throws() new Column(columnName2, DbType.Int32) ); - Assert.Throws(() => Provider.AddIndex(tableName, - new Index - { - Name = indexName, - KeyColumns = [columnName1], - FilterItems = [new FilterItem { Filter = FilterType.GreaterThan, ColumnName = columnName2, Value = 12 }] - })); + var definition = new Index + { + Name = indexName, + KeyColumns = [columnName1], + FilterItems = [new FilterItem { Filter = FilterType.GreaterThan, ColumnName = columnName2, Value = 12 }] + }; + if (Provider is OracleTransformationProvider) + Assert.Throws(() => Provider.AddIndex(tableName, definition)); + else + { + Provider.AddIndex(tableName, definition); + var actual = Provider.GetIndexes(tableName).Single(); + Assert.That(actual.KeyColumns, Is.EqualTo(definition.KeyColumns).IgnoreCase); + Assert.That(actual.FilterItems.Single().ColumnName, Is.EqualTo(columnName2).IgnoreCase); + Assert.That(actual.FilterItems.Single().Value, Is.EqualTo(12)); + } } [Test] diff --git a/src/Migrator.Tests/Providers/OracleProvider/OracleTransformationProvider_AddIndexTests.cs b/src/Migrator.Tests/Providers/OracleProvider/OracleTransformationProvider_AddIndexTests.cs index c9613a5b..1531eca3 100644 --- a/src/Migrator.Tests/Providers/OracleProvider/OracleTransformationProvider_AddIndexTests.cs +++ b/src/Migrator.Tests/Providers/OracleProvider/OracleTransformationProvider_AddIndexTests.cs @@ -169,7 +169,7 @@ public void AddIndex_FilterItemsCombinedWithUnique_Throws() ]; // Act/Assert - Assert.Throws(() => Provider.AddIndex(tableName, + Assert.Throws(() => Provider.AddIndex(tableName, new Index { Name = indexName, diff --git a/src/Migrator.Tests/Providers/PostgreSQL/PostgreSQLTransformationProvider_AddIndexTests.cs b/src/Migrator.Tests/Providers/PostgreSQL/PostgreSQLTransformationProvider_AddIndexTests.cs index db1d7070..7ffbf908 100644 --- a/src/Migrator.Tests/Providers/PostgreSQL/PostgreSQLTransformationProvider_AddIndexTests.cs +++ b/src/Migrator.Tests/Providers/PostgreSQL/PostgreSQLTransformationProvider_AddIndexTests.cs @@ -23,6 +23,11 @@ public async Task SetUpAsync() { await BeginPostgreSQLTransactionAsync(); } + + [TestCase(false)] + [TestCase(true)] + public void FilteredIndexWithNonKeyColumnAndNullPredicateRoundTrips(bool fluent) + => FilteredIndexTests.VerifyRoundTrip(Provider, fluent, includeColumns: true); [Test] public void AddTableWithCompoundPrimaryKey() diff --git a/src/Migrator.Tests/Providers/SQLServer/SQLServerTransformationProvider_AddIndexTests.cs b/src/Migrator.Tests/Providers/SQLServer/SQLServerTransformationProvider_AddIndexTests.cs index e319a2a3..138ca503 100644 --- a/src/Migrator.Tests/Providers/SQLServer/SQLServerTransformationProvider_AddIndexTests.cs +++ b/src/Migrator.Tests/Providers/SQLServer/SQLServerTransformationProvider_AddIndexTests.cs @@ -22,6 +22,11 @@ public async Task SetUpAsync() { await BeginSQLServerTransactionAsync(); } + + [TestCase(false)] + [TestCase(true)] + public void FilteredIndexWithNonKeyColumnAndNullPredicateRoundTrips(bool fluent) + => FilteredIndexTests.VerifyRoundTrip(Provider, fluent, includeColumns: true); [Test] public void AddIndex_Unique_Success() diff --git a/src/Migrator/Framework/Fluent/Operations.cs b/src/Migrator/Framework/Fluent/Operations.cs index 18075394..b5d5ffda 100644 --- a/src/Migrator/Framework/Fluent/Operations.cs +++ b/src/Migrator/Framework/Fluent/Operations.cs @@ -256,7 +256,14 @@ public static class Definitions public static IDbField Copy(IDbField field) => field switch { Column c => CopyColumn(c), - Index i => new Index { Name = i.Name, Unique = i.Unique, Clustered = i.Clustered, KeyColumns = (string[])i.KeyColumns.Clone(), IncludeColumns = (string[])i.IncludeColumns.Clone(), FilterItems = i.FilterItems.Select(f => new DotNetProjects.Migrator.Providers.Models.Indexes.FilterItem { ColumnName = f.ColumnName, Filter = f.Filter, Value = f.Value }).ToList() }, + Index i => new Index + { + Name = i.Name, Unique = i.Unique, Clustered = i.Clustered, + PrimaryKey = i.PrimaryKey, UniqueConstraint = i.UniqueConstraint, + UnsupportedFilterBehavior = i.UnsupportedFilterBehavior, + KeyColumns = i.KeyColumns?.ToArray() ?? [], IncludeColumns = i.IncludeColumns?.ToArray() ?? [], + FilterItems = i.FilterItems?.Select(f => new DotNetProjects.Migrator.Providers.Models.Indexes.FilterItem { ColumnName = f.ColumnName, Filter = f.Filter, Value = f.Value }).ToList() ?? [] + }, ForeignKeyConstraint f => new ForeignKeyConstraint(f.Name, f.ParentTable, (string[])f.ParentColumns.Clone(), f.ChildTable, (string[])f.ChildColumns.Clone()) { OnDelete = f.OnDelete, OnUpdate = f.OnUpdate, Match = f.Match, Id = f.Id }, PrimaryKeyConstraint k => new PrimaryKeyConstraint(k.Name, k.KeyColumns) { NonClustered = k.NonClustered }, UniqueConstraint u => new UniqueConstraint { Name = u.Name, KeyColumns = (string[])u.KeyColumns.Clone() }, diff --git a/src/Migrator/Framework/Fluent/SchemaBuilders.cs b/src/Migrator/Framework/Fluent/SchemaBuilders.cs index cc784475..5bd69b59 100644 --- a/src/Migrator/Framework/Fluent/SchemaBuilders.cs +++ b/src/Migrator/Framework/Fluent/SchemaBuilders.cs @@ -108,6 +108,14 @@ public IndexOptionsBuilder WithFilter(params FilterItem[] filters) index.FilterItems = filters.Select(f => new FilterItem { ColumnName = f.ColumnName, Filter = f.Filter, Value = f.Value }).ToList(); return this; } + + /// Choose whether unsupported filters throw or are omitted from the index. + public IndexOptionsBuilder OnUnsupportedFilter(UnsupportedIndexFilterBehavior behavior) + { + if (!Enum.IsDefined(behavior)) throw new ArgumentOutOfRangeException(nameof(behavior)); + index.UnsupportedFilterBehavior = behavior; + return this; + } } public sealed class IndexDefinitionOnTableBuilder diff --git a/src/Migrator/Framework/Index.cs b/src/Migrator/Framework/Index.cs index 643a8cc3..de4f26f7 100644 --- a/src/Migrator/Framework/Index.cs +++ b/src/Migrator/Framework/Index.cs @@ -38,7 +38,15 @@ public class Index : IDbField /// /// Gets or sets items that represent filter expressions in filtered indexes. Currently string, integer and boolean values are supported. - /// Attention: In SQL Server the column used in the filter must be NOT NULL. + /// Filter columns need not be key columns on SQL Server, PostgreSQL and SQLite. + /// EqualTo/NotEqualTo with null or DBNull.Value generate IS NULL/IS NOT NULL. /// public List FilterItems { get; set; } = []; + + /// + /// Controls unsupported filters. The default is to throw. Ignore creates an unfiltered + /// index, so a unique index will enforce uniqueness across all rows. + /// This option does not suppress invalid definitions or database errors. + /// + public UnsupportedIndexFilterBehavior UnsupportedFilterBehavior { get; set; } = UnsupportedIndexFilterBehavior.Throw; } diff --git a/src/Migrator/Framework/UnsupportedIndexFilterBehavior.cs b/src/Migrator/Framework/UnsupportedIndexFilterBehavior.cs new file mode 100644 index 00000000..cc2b80ba --- /dev/null +++ b/src/Migrator/Framework/UnsupportedIndexFilterBehavior.cs @@ -0,0 +1,11 @@ +namespace DotNetProjects.Migrator.Framework; + +/// Controls index creation when the provider cannot apply the requested filters. +public enum UnsupportedIndexFilterBehavior +{ + /// Fail before creating the index. + Throw, + + /// Create the index without any filters. Unique indexes then constrain all rows. + Ignore +} diff --git a/src/Migrator/Providers/Impl/DB2/DB2TransformationProvider.cs b/src/Migrator/Providers/Impl/DB2/DB2TransformationProvider.cs index c657fd41..4a262e49 100644 --- a/src/Migrator/Providers/Impl/DB2/DB2TransformationProvider.cs +++ b/src/Migrator/Providers/Impl/DB2/DB2TransformationProvider.cs @@ -127,8 +127,9 @@ public override void RemoveAllIndexes(string table) public override bool IndexExists(string table, string name) => GetIndexes(table).Any(i => i.Name == Name(name)); public override string AddIndex(string table, Index index) { + ShouldApplyIndexFilters(index, supported: false); if (index.KeyColumns.Length == 0) throw new ArgumentException("An index needs key columns.", nameof(index)); - if (index.IncludeColumns.Length != 0 || index.FilterItems.Count != 0 || index.Clustered) + if (index.IncludeColumns.Length != 0 || index.Clustered) throw new NotSupportedException("This Db2 provider supports ordinary and unique indexes without INCLUDE, filters or clustering."); var name = index.Name ?? $"IX_{table}_{string.Join("_", index.KeyColumns)}"; ExecuteNonQuery($"CREATE {(index.Unique ? "UNIQUE " : "")}INDEX {Identifier(name)} ON {Identifier(table)} ({string.Join(", ", index.KeyColumns.Select(Identifier))})"); diff --git a/src/Migrator/Providers/Impl/Firebird/FirebirdTransformationProvider.cs b/src/Migrator/Providers/Impl/Firebird/FirebirdTransformationProvider.cs index ac1ddbb5..a0261b46 100644 --- a/src/Migrator/Providers/Impl/Firebird/FirebirdTransformationProvider.cs +++ b/src/Migrator/Providers/Impl/Firebird/FirebirdTransformationProvider.cs @@ -171,8 +171,9 @@ public override void ChangeColumn(string table, Column column) public override string AddIndex(string table, Index index) { + ShouldApplyIndexFilters(index, supported: false); if (index.KeyColumns.Length == 0) throw new ArgumentException("An index needs key columns.", nameof(index)); - if (index.IncludeColumns.Length != 0 || index.FilterItems.Count != 0 || index.Clustered) + if (index.IncludeColumns.Length != 0 || index.Clustered) throw new NotSupportedException("This Firebird provider supports ordinary and unique indexes without INCLUDE or filters."); var name = index.Name ?? $"IX_{table}_{string.Join("_", index.KeyColumns)}"; ExecuteNonQuery($"CREATE {(index.Unique ? "UNIQUE " : "")}INDEX {QuoteConstraintNameIfRequired(name)} ON {QuoteTableNameIfRequired(table)} ({string.Join(", ", index.KeyColumns.Select(QuoteColumnNameIfRequired))})"); diff --git a/src/Migrator/Providers/Impl/Hana/HanaTransformationProvider.cs b/src/Migrator/Providers/Impl/Hana/HanaTransformationProvider.cs index 7c534561..bd22e01f 100644 --- a/src/Migrator/Providers/Impl/Hana/HanaTransformationProvider.cs +++ b/src/Migrator/Providers/Impl/Hana/HanaTransformationProvider.cs @@ -166,7 +166,8 @@ public override ForeignKeyConstraint[] GetForeignKeyConstraints(string table) } public override string AddIndex(string table, Index index) { - if (index.Clustered || index.IncludeColumns?.Length > 0 || index.FilterItems?.Count > 0) + ShouldApplyIndexFilters(index, supported: false); + if (index.Clustered || index.IncludeColumns?.Length > 0) throw new NotSupportedException("HANA index INCLUDE, clustered and filtered options are not supported by this provider."); if (index.KeyColumns?.Length is not > 0) throw new ArgumentException("Index key columns are required.", nameof(index)); var name = index.Name ?? "IX_" + Name(table).Table + "_" + string.Join("_", index.KeyColumns); diff --git a/src/Migrator/Providers/Impl/Informix/InformixTransformationProvider.cs b/src/Migrator/Providers/Impl/Informix/InformixTransformationProvider.cs index 4e8f471d..7ddd294f 100644 --- a/src/Migrator/Providers/Impl/Informix/InformixTransformationProvider.cs +++ b/src/Migrator/Providers/Impl/Informix/InformixTransformationProvider.cs @@ -264,8 +264,9 @@ public override void RemoveAllIndexes(string table) public override bool IndexExists(string table, string name) => GetIndexes(table).Any(i => i.Name == Name(name)); public override string AddIndex(string table, Index index) { + ShouldApplyIndexFilters(index, supported: false); if (index.KeyColumns.Length == 0) throw new ArgumentException("An index needs key columns.", nameof(index)); - if (index.IncludeColumns.Length != 0 || index.FilterItems.Count != 0 || index.Clustered) + if (index.IncludeColumns.Length != 0 || index.Clustered) throw new NotSupportedException("This Informix provider supports ordinary and unique indexes without INCLUDE, filters or clustering."); var name = index.Name ?? $"ix_{table}_{string.Join("_", index.KeyColumns)}"; ExecuteNonQuery($"CREATE {(index.Unique ? "UNIQUE " : "")}INDEX {name} ON {table} ({string.Join(", ", index.KeyColumns)})"); diff --git a/src/Migrator/Providers/Impl/Mysql/MySqlTransformationProvider.cs b/src/Migrator/Providers/Impl/Mysql/MySqlTransformationProvider.cs index 9162e48a..e78b8827 100644 --- a/src/Migrator/Providers/Impl/Mysql/MySqlTransformationProvider.cs +++ b/src/Migrator/Providers/Impl/Mysql/MySqlTransformationProvider.cs @@ -334,8 +334,9 @@ public override bool ViewExists(string view) => public override string AddIndex(string table, Index index) { + ShouldApplyIndexFilters(index, supported: false); if (index.KeyColumns.Length == 0) throw new ArgumentException("An index needs key columns.", nameof(index)); - if (index.IncludeColumns.Length != 0 || index.FilterItems.Count != 0 || index.Clustered) + if (index.IncludeColumns.Length != 0 || index.Clustered) throw new NotSupportedException("MySQL and MariaDB do not support included columns, filtered indexes or explicit clustered indexes."); var name = index.Name ?? $"IX_{table}_{string.Join("_", index.KeyColumns)}"; ExecuteNonQuery($"CREATE {(index.Unique ? "UNIQUE " : "")}INDEX {_dialect.QuoteIdentifier(name)} ON {_dialect.Quote(table)} ({string.Join(", ", index.KeyColumns.Select(_dialect.Quote))})"); diff --git a/src/Migrator/Providers/Impl/Oracle/OracleTransformationProvider.cs b/src/Migrator/Providers/Impl/Oracle/OracleTransformationProvider.cs index 5c123a28..95550372 100644 --- a/src/Migrator/Providers/Impl/Oracle/OracleTransformationProvider.cs +++ b/src/Migrator/Providers/Impl/Oracle/OracleTransformationProvider.cs @@ -70,64 +70,28 @@ public override void AddForeignKey(string name, string primaryTable, string[] pr public override string AddIndex(string table, Index index) { - ValidateIndex(tableName: table, index: index); - var hasFilterItems = index.FilterItems != null && index.FilterItems.Count > 0; + var hasFilterItems = ShouldApplyIndexFilters(index, + supported: !index.Unique && (index.FilterItems == null || index.FilterItems.All(f => + index.KeyColumns.Any(c => c.Equals(f.ColumnName, StringComparison.OrdinalIgnoreCase)))), + reason: "Oracle filter emulation requires a non-unique index and filters on key columns only."); + ValidateIndex(table, index, validateFilters: hasFilterItems); if (index.IncludeColumns?.Length > 0 || index.Clustered) throw new NotSupportedException("Oracle does not support included columns or SQL Server-style clustered indexes. Use an explicit Oracle operation."); - if (index.Unique && hasFilterItems) - { - throw new MigrationException($"You cannot use unique together with functional expressions in Oracle ({nameof(FilterItem)})."); - } - var name = QuoteConstraintNameIfRequired(index.Name); table = QuoteTableNameIfRequired(table); + var keyColumns = index.KeyColumns; List singleFilterStrings = []; - - if (hasFilterItems) { - // In Oracle functional expressions replace the normal columns so we need to remove them - if (index.KeyColumns != null && index.KeyColumns.Length > 0) - { - var keyColumnsList = index.KeyColumns.ToList(); - - for (var i = keyColumnsList.Count - 1; i >= 0; i--) - { - if (index.FilterItems.Any(x => keyColumnsList[i].Equals(x.ColumnName, StringComparison.OrdinalIgnoreCase))) - { - keyColumnsList.RemoveAt(i); - } - } - - index.KeyColumns = keyColumnsList.ToArray(); - } - - foreach (var filterItem in index.FilterItems) - { - var comparisonString = _dialect.GetComparisonStringByFilterType(filterItem.Filter); - - var filterColumnQuoted = QuoteColumnNameIfRequired(filterItem.ColumnName); - string value = null; - - value = filterItem.Value switch - { - bool booleanValue => booleanValue ? "TRUE" : "FALSE", - string stringValue => $"'{stringValue.Replace("'", "''")}'", - byte or short or int or long => Convert.ToInt64(filterItem.Value).ToString(), - sbyte or ushort or uint or ulong => Convert.ToUInt64(filterItem.Value).ToString(), - _ => throw new NotImplementedException($"Given type in '{nameof(FilterItem)}' is not implemented. Please file an issue."), - }; - - var singleFilterString = $"CASE WHEN {filterColumnQuoted} {comparisonString} {value} THEN {filterColumnQuoted} ELSE NULL END"; - - singleFilterStrings.Add(singleFilterString); - } + keyColumns = keyColumns.Where(c => !index.FilterItems.Any(f => c.Equals(f.ColumnName, StringComparison.OrdinalIgnoreCase))).ToArray(); + foreach (var filter in index.FilterItems) + singleFilterStrings.Add($"CASE WHEN {IndexFilterSql.Format(_dialect, filter, numericBooleans: false)} THEN {QuoteColumnNameIfRequired(filter.ColumnName)} ELSE NULL END"); } - var mixedColumnNamesAndFilters = QuoteColumnNamesIfRequired(index.KeyColumns).ToList(); + var mixedColumnNamesAndFilters = QuoteColumnNamesIfRequired(keyColumns).ToList(); mixedColumnNamesAndFilters.AddRange(singleFilterStrings); var columnNamesAndFiltersString = $"({string.Join(", ", mixedColumnNamesAndFilters)})"; diff --git a/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLTransformationProvider.cs b/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLTransformationProvider.cs index 687e7aa9..35866443 100644 --- a/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLTransformationProvider.cs +++ b/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLTransformationProvider.cs @@ -66,7 +66,8 @@ protected override string GetPrimaryKeyConstraintName(string table) public override string AddIndex(string table, Index index) { - ValidateIndex(tableName: table, index: index); + var hasFilterItems = ShouldApplyIndexFilters(index, supported: true); + ValidateIndex(table, index, validateFilters: hasFilterItems); var hasIncludedColumns = index.IncludeColumns != null && index.IncludeColumns.Length > 0; var name = QuoteConstraintNameIfRequired(index.Name); @@ -85,33 +86,8 @@ public override string AddIndex(string table, Index index) includeString = $"INCLUDE ({string.Join(", ", includeColumnsQuoted)})"; } - if (index.FilterItems != null && index.FilterItems.Count > 0) - { - List singleFilterStrings = []; - - foreach (var filterItem in index.FilterItems) - { - var comparisonString = _dialect.GetComparisonStringByFilterType(filterItem.Filter); - - var filterColumnQuoted = QuoteColumnNameIfRequired(filterItem.ColumnName); - string value = null; - - value = filterItem.Value switch - { - bool booleanValue => booleanValue ? "TRUE" : "FALSE", - string stringValue => $"'{stringValue.Replace("'", "''")}'", - byte or short or int or long => Convert.ToInt64(filterItem.Value).ToString(), - sbyte or ushort or uint or ulong => Convert.ToUInt64(filterItem.Value).ToString(), - _ => throw new NotImplementedException($"Given type in '{nameof(FilterItem)}' is not implemented. Please file an issue."), - }; - - var singleFilterString = $"{filterColumnQuoted} {comparisonString} {value}"; - - singleFilterStrings.Add(singleFilterString); - } - - filterString = $"WHERE {string.Join(" AND ", singleFilterStrings)}"; - } + if (hasFilterItems) + filterString = "WHERE " + string.Join(" AND ", index.FilterItems.Select(f => IndexFilterSql.Format(_dialect, f, numericBooleans: false))); List list = []; list.Add("CREATE"); @@ -121,8 +97,8 @@ public override string AddIndex(string table, Index index) list.Add("ON"); list.Add(table); list.Add(columnsString); - list.Add(filterString); list.Add(includeString); + list.Add(filterString); var sql = string.Join(" ", list.Where(x => !string.IsNullOrWhiteSpace(x))); @@ -135,9 +111,6 @@ public override Index[] GetIndexes(string table) { var columns = GetColumns(table); - // Since the migrator does not support schemas at this point in time we set the schema to "public" - var schemaName = "public"; - var indexes = new List(); var sql = @$" @@ -151,7 +124,7 @@ public override Index[] GetIndexes(string table) con.contype = 'p' AS is_primary_constraint, pg_get_indexdef(idx.indexrelid) AS index_definition, ( - SELECT string_agg(att.attname, ', ') + SELECT string_agg(att.attname, ', ' ORDER BY cols.ord) FROM unnest(idx.indkey) WITH ORDINALITY AS cols(attnum, ord) JOIN pg_attribute att ON att.attrelid = idx.indrelid @@ -159,7 +132,7 @@ JOIN pg_attribute att WHERE cols.ord <= idx.indnkeyatts ) AS index_columns, ( - SELECT string_agg(att.attname, ', ') + SELECT string_agg(att.attname, ', ' ORDER BY cols.ord) FROM unnest(idx.indkey) WITH ORDINALITY AS cols(attnum, ord) JOIN pg_attribute att ON att.attrelid = idx.indrelid @@ -171,13 +144,11 @@ FROM pg_index idx JOIN pg_class cls ON cls.oid = idx.indexrelid JOIN pg_class tbl ON tbl.oid = idx.indrelid JOIN pg_namespace nsp ON nsp.oid = tbl.relnamespace - LEFT JOIN pg_constraint con ON con.conindid = idx.indexrelid - WHERE - lower(tbl.relname) = '{table.ToLowerInvariant()}' AND - nsp.nspname = '{schemaName}'"; + LEFT JOIN pg_constraint con ON con.conindid = idx.indexrelid AND con.conrelid = idx.indrelid + WHERE idx.indrelid = to_regclass(@relation)"; - using (var cmd = CreateCommand()) - using (var reader = ExecuteQuery(cmd, string.Format(sql, table))) + using (var cmd = MetadataCommand(table)) + using (var reader = ExecuteQuery(cmd, sql)) { var includeColumnsOrdinal = reader.GetOrdinal("include_columns"); var indexColumnsOrdinal = reader.GetOrdinal("index_columns"); @@ -199,86 +170,7 @@ FROM pg_index idx var indexColumns = !reader.IsDBNull(indexColumnsOrdinal) ? reader.GetString(indexColumnsOrdinal) : null; var indexDefinition = reader.GetString(indexDefinitionOrdinal); var partialColumns = !reader.IsDBNull(partialFilterOrdinal) ? reader.GetString(partialFilterOrdinal) : null; - List filterItems = []; - - if (!string.IsNullOrWhiteSpace(partialColumns)) - { - partialColumns = partialColumns.Substring(1, partialColumns.Length - 2); - var comparisonStrings = _dialect.GetComparisonStrings(); - var partialSplitted = Regex.Split(partialColumns, " AND ").Select(x => x.Trim()).ToList(); - - if (partialSplitted.Count > 1) - { - partialSplitted = partialSplitted.Select(x => x.Substring(1, x.Length - 2)).ToList(); - } - - foreach (var partialItemString in partialSplitted) - { - string[] splits = []; - var filterType = FilterType.None; - - foreach (var comparisonString in comparisonStrings.OrderByDescending(x => x)) - { - splits = Regex.Split(partialItemString, $" {comparisonString} "); - - if (splits.Length == 2) - { - filterType = _dialect.GetFilterTypeByComparisonString(comparisonString); - break; - } - } - - if (splits.Length != 2) - { - throw new NotImplementedException($"Comparison string not found in '{partialItemString}'"); - } - - var columnNameString = splits[0]; - var columnNameRegex = new Regex(@"(?<=^\().+(?=\)::(text|boolean|integer)$)"); - - if (columnNameRegex.Match(columnNameString) is Match matchColumnName && matchColumnName.Success) - { - columnNameString = matchColumnName.Value; - } - - var column = columns.First(x => columnNameString.Equals(x.Name, StringComparison.OrdinalIgnoreCase)); - var valueAsString = splits[1]; - var stringValueNumericRegex = new Regex(@"(?<=^\()[^\)]+(?=\)::numeric$)"); - - if (stringValueNumericRegex.Match(valueAsString) is Match valueNumericMatch && valueNumericMatch.Success) - { - valueAsString = valueNumericMatch.Value; - } - - var stringValueRegex = new Regex("(?<=^').+(?='::(text|boolean|integer|bigint)$)"); - - if (stringValueRegex.Match(valueAsString) is Match match && match.Success) - { - valueAsString = match.Value; - } - - var filterItem = new FilterItem - { - ColumnName = column.Name, - Filter = filterType, - Value = column.MigratorDbType switch - { - MigratorDbType.Int16 => short.Parse(valueAsString), - MigratorDbType.Int32 => int.Parse(valueAsString), - MigratorDbType.Int64 => long.Parse(valueAsString), - MigratorDbType.UInt16 => ushort.Parse(valueAsString), - MigratorDbType.UInt32 => uint.Parse(valueAsString), - MigratorDbType.UInt64 => ulong.Parse(valueAsString), - MigratorDbType.Decimal => decimal.Parse(valueAsString), - MigratorDbType.Boolean => valueAsString == "1" || valueAsString.Equals("true", StringComparison.OrdinalIgnoreCase), - MigratorDbType.String => valueAsString, - _ => throw new NotImplementedException($"Type '{column.MigratorDbType}' not yet supported - there are many variations. Please file an issue."), - } - }; - - filterItems.Add(filterItem); - } - } + var filterItems = IndexFilterSql.Parse(partialColumns, columns); var index = new Index { diff --git a/src/Migrator/Providers/Impl/SQLite/SQLiteTransformationProvider.cs b/src/Migrator/Providers/Impl/SQLite/SQLiteTransformationProvider.cs index 923f98bb..0858ff8b 100644 --- a/src/Migrator/Providers/Impl/SQLite/SQLiteTransformationProvider.cs +++ b/src/Migrator/Providers/Impl/SQLite/SQLiteTransformationProvider.cs @@ -437,19 +437,22 @@ public override void RemoveAllForeignKeys(string tableName, string columnName) } public string[] GetCreateIndexSqlStrings(string table) + => GetCreateIndexSqlByName(table).Values.ToArray(); + + private Dictionary GetCreateIndexSqlByName(string table) { - var sqlStrings = new List(); + var sqlStrings = new Dictionary(StringComparer.OrdinalIgnoreCase); using (var cmd = CreateCommand()) - using (var reader = ExecuteQuery(cmd, string.Format("SELECT sql FROM sqlite_master WHERE type='index' AND sql NOT NULL AND lower(tbl_name)=lower('{0}')", table))) + using (var reader = ExecuteQuery(cmd, string.Format("SELECT name, sql FROM sqlite_master WHERE type='index' AND sql NOT NULL AND lower(tbl_name)=lower('{0}')", table.Replace("'", "''")))) { while (reader.Read()) { - sqlStrings.Add((string)reader[0]); + sqlStrings.Add(reader.GetString(0), reader.GetString(1)); } } - return [.. sqlStrings]; + return sqlStrings; } public void MoveIndexesFromOriginalTable(string origTable, string newTable) @@ -1243,10 +1246,9 @@ public override bool IndexExists(string table, string name) public override Index[] GetIndexes(string table) { - var afterWhereRegex = new Regex("(?<= WHERE ).+"); List indexes = []; - var indexCreateScripts = GetCreateIndexSqlStrings(table); + var indexCreateScripts = GetCreateIndexSqlByName(table); var pragmaIndexListItems = GetPragmaIndexListItems(table).Where(x => x.Origin == "c"); @@ -1273,68 +1275,8 @@ public override Index[] GetIndexes(string table) Unique = pragmaIndexListItem.Unique }; - var script = indexCreateScripts.FirstOrDefault(x => x.Contains(pragmaIndexListItem.Name, StringComparison.OrdinalIgnoreCase)); - - if (script != null) - { - if (afterWhereRegex.Match(script) is Match match && match.Success) - { - // We cannot use GeneratedRegexAttribute due to old .NET version - var andSplitted = Regex.Split(match.Value, " AND "); - - var filterSingleStrings = andSplitted - .Select(x => x.Trim()) - .ToList(); - - foreach (var filterSingleString in filterSingleStrings) - { - var splitted = filterSingleString.Split(' ') - .Where(x => !string.IsNullOrWhiteSpace(x)) - .Select(x => x.Trim()) - .ToList(); - - var filterItem = new FilterItem { ColumnName = splitted[0], Filter = _dialect.GetFilterTypeByComparisonString(splitted[1]) }; - - var column = columns.Single(x => x.Name.Equals(splitted[0], StringComparison.OrdinalIgnoreCase)); - - var sqliteIntegerDataTypes = new[] { - MigratorDbType.Int16, - MigratorDbType.Int32, - MigratorDbType.Int64, - MigratorDbType.UInt16, - MigratorDbType.UInt32, - MigratorDbType.UInt64 - }; - - if (sqliteIntegerDataTypes.Contains(column.MigratorDbType)) - { - if (long.TryParse(splitted[2], out var longValue)) - { - filterItem.Value = longValue; - } - else if (ulong.TryParse(splitted[2], out var uLongValue)) - { - filterItem.Value = uLongValue; - } - else - { - throw new Exception(); - } - } - else - { - filterItem.Value = column.MigratorDbType switch - { - MigratorDbType.Boolean => splitted[2] == "1" || splitted[2].Equals("true", StringComparison.OrdinalIgnoreCase), - MigratorDbType.String => splitted[2].Substring(1, splitted[2].Length - 2), - _ => throw new NotImplementedException("Type not yet supported. Please file an issue."), - }; - } - - index.FilterItems.Add(filterItem); - } - } - } + if (indexCreateScripts.TryGetValue(pragmaIndexListItem.Name, out var script)) + index.FilterItems = IndexFilterSql.ParseCreateIndex(script, columns); indexes.Add(index); } @@ -1352,7 +1294,8 @@ public override void AddTable(string name, string engine, params IDbField[] fiel public override string AddIndex(string table, Index index) { - ValidateIndex(table, index); + var hasFilterItems = ShouldApplyIndexFilters(index, supported: true); + ValidateIndex(table, index, validateFilters: hasFilterItems); var hasIncludedColumns = index.IncludeColumns != null && index.IncludeColumns.Length > 0; @@ -1375,38 +1318,8 @@ public override string AddIndex(string table, Index index) var columnsString = $"({string.Join(", ", columns)})"; var filterString = string.Empty; - if (index.FilterItems != null && index.FilterItems.Count > 0) - { - List singleFilterStrings = []; - - foreach (var filterItem in index.FilterItems) - { - var comparisonString = _dialect.GetComparisonStringByFilterType(filterItem.Filter); - - var filterColumnQuoted = QuoteColumnNameIfRequired(filterItem.ColumnName); - string value = null; - - value = filterItem.Value switch - { - bool booleanValue => booleanValue ? "1" : "0", - string stringValue => $"'{stringValue.Replace("'", "''")}'", - byte or short or int or long => Convert.ToInt64(filterItem.Value).ToString(), - sbyte or ushort or uint or ulong => Convert.ToUInt64(filterItem.Value).ToString(), - _ => throw new NotImplementedException("Given type is not implemented. Please file an issue."), - }; - - if ((filterItem.Value is string || filterItem.Value is bool) && filterItem.Filter != FilterType.EqualTo && filterItem.Filter != FilterType.NotEqualTo) - { - throw new MigrationException($"Bool and string in {nameof(FilterItem)} can only be used with '{nameof(FilterType.EqualTo)}' or '{nameof(FilterType.EqualTo)}'."); - } - - var singleFilterString = $"{filterColumnQuoted} {comparisonString} {value}"; - - singleFilterStrings.Add(singleFilterString); - } - - filterString = $"WHERE {string.Join(" AND ", singleFilterStrings)}"; - } + if (hasFilterItems) + filterString = "WHERE " + string.Join(" AND ", index.FilterItems.Select(f => IndexFilterSql.Format(_dialect, f, numericBooleans: true))); List list = ["CREATE", uniqueString, "INDEX", name, "ON", table, columnsString, filterString]; diff --git a/src/Migrator/Providers/Impl/SqlServer/SqlServerTransformationProvider.cs b/src/Migrator/Providers/Impl/SqlServer/SqlServerTransformationProvider.cs index d3775b31..6f704ada 100644 --- a/src/Migrator/Providers/Impl/SqlServer/SqlServerTransformationProvider.cs +++ b/src/Migrator/Providers/Impl/SqlServer/SqlServerTransformationProvider.cs @@ -172,7 +172,8 @@ public override void AddPrimaryKeyNonClustered(string name, string table, params public override string AddIndex(string table, Index index) { - ValidateIndex(tableName: table, index: index); + var hasFilterItems = ShouldApplyIndexFilters(index, supported: _dialect is not SqlServer2005Dialect); + ValidateIndex(table, index, validateFilters: hasFilterItems); var hasIncludedColumns = index.IncludeColumns != null && index.IncludeColumns.Length > 0; var name = QuoteConstraintNameIfRequired(index.Name); @@ -181,49 +182,12 @@ public override string AddIndex(string table, Index index) var uniqueString = index.Unique ? "UNIQUE" : null; var columnsString = $"({string.Join(", ", columns)})"; - var includeString = hasIncludedColumns ? $"INCLUDE ({string.Join(", ", index.IncludeColumns)})" : null; + var includeString = hasIncludedColumns ? $"INCLUDE ({string.Join(", ", QuoteColumnNamesIfRequired(index.IncludeColumns))})" : null; var filterString = string.Empty; var clusteredString = index.Clustered ? "CLUSTERED" : "NONCLUSTERED"; - if (index.FilterItems != null && index.FilterItems.Count > 0) - { - List singleFilterStrings = []; - - foreach (var filterItem in index.FilterItems) - { - var comparisonString = _dialect.GetComparisonStringByFilterType(filterItem.Filter); - - var filterColumnQuoted = QuoteColumnNameIfRequired(filterItem.ColumnName); - string value = null; - - if (filterItem.Value is bool booleanValue) - { - value = booleanValue ? "1" : "0"; - } - else if (filterItem.Value is string stringValue) - { - value = $"'{stringValue}'"; - } - else if (filterItem.Value is byte || filterItem.Value is short || filterItem.Value is int || filterItem.Value is long) - { - value = Convert.ToInt64(filterItem.Value).ToString(); - } - else if (filterItem.Value is sbyte || filterItem.Value is ushort || filterItem.Value is uint || filterItem.Value is ulong) - { - value = Convert.ToUInt64(filterItem.Value).ToString(); - } - else - { - throw new NotImplementedException("Given type is not implemented. Please file an issue."); - } - - var singleFilterString = $"{filterColumnQuoted} {comparisonString} {value}"; - - singleFilterStrings.Add(singleFilterString); - } - - filterString = $"WHERE {string.Join(" AND ", singleFilterStrings)}"; - } + if (hasFilterItems) + filterString = "WHERE " + string.Join(" AND ", index.FilterItems.Select(f => IndexFilterSql.Format(_dialect, f, numericBooleans: true))); List list = []; list.Add("CREATE"); @@ -302,7 +266,8 @@ public override void RemoveColumnDefaultValue(string table, string column) public override Index[] GetIndexes(string table) { - var relation = SqlIdentifier.Catalog(QuoteTableNameIfRequired(table)); + var qualifiedTable = QuoteTableNameIfRequired(table); + var relation = SqlIdentifier.Catalog(qualifiedTable); var schemaName = relation.Schema ?? "dbo"; table = relation.Name; @@ -316,7 +281,7 @@ public override Index[] GetIndexes(string table) i.is_unique AS IsUnique, i.is_primary_key AS IsPrimaryKey, i.is_unique_constraint AS IsUniqueConstraint, - ic.index_column_id AS ColumnOrder, + CASE WHEN ic.is_included_column = 1 THEN ic.index_column_id ELSE ic.key_ordinal END AS ColumnOrder, col.name AS ColumnName, ic.is_descending_key AS IsDescending, ic.is_included_column AS IsIncludedColumn, @@ -386,75 +351,8 @@ ORDER BY { var first = indexGroup.First(); - List filterItems = []; - - if (!string.IsNullOrWhiteSpace(first.FilterString)) - { - const string unexpectedPatternString = "Unexpected pattern in filter string detected. Not implemented yet - please file an issue"; - var comparisonStrings = _dialect.GetComparisonStrings(); - var stripOuterBracesRegex = new Regex(@"(?<=^\().+(?=\)$)"); - var stripBracesMatch = stripOuterBracesRegex.Match(first.FilterString.Trim()); - - if (!stripBracesMatch.Success) - { - throw new NotImplementedException(unexpectedPatternString); - } - - var andSplitted = Regex.Split(stripBracesMatch.Value, @" AND (?=\[)") - .Select(x => x.Trim()) - .ToList(); - - var columns = GetColumns(table: table); - - foreach (var andSplittedItem in andSplitted) - { - var filterItem = new FilterItem(); - // We assume nobody uses column names with brackets in it. - var columnRegex = new Regex(@"(?<=^\[)[^\]]+"); - var columnMatch = columnRegex.Match(andSplittedItem); - - if (!columnMatch.Success) - { - throw new NotImplementedException(unexpectedPatternString); - } - - filterItem.ColumnName = columnMatch.Value; - var column = columns.OrderByDescending(x => x.Name).First(x => x.Name.Equals(filterItem.ColumnName, StringComparison.OrdinalIgnoreCase)); - - var remainingString = andSplittedItem.Substring(filterItem.ColumnName.Length + 2); - var comparisonString = comparisonStrings.OrderByDescending(x => x.Length) - .First(x => remainingString.StartsWith(x)); - - filterItem.Filter = _dialect.GetFilterTypeByComparisonString(comparisonString); - remainingString = remainingString.Substring(comparisonString.Length); - - var valueRegex = new Regex(@"(?<=^[\(|']).+(?=[\)|']$)"); - var valueStringMatch = valueRegex.Match(remainingString); - - if (!valueStringMatch.Success) - { - throw new NotImplementedException(unexpectedPatternString); - } - - var valueAsString = valueStringMatch.Value; - - filterItem.Value = column.MigratorDbType switch - { - MigratorDbType.Int16 => short.Parse(valueAsString), - MigratorDbType.Int32 => int.Parse(valueAsString), - MigratorDbType.Int64 => long.Parse(valueAsString), - MigratorDbType.UInt16 => ushort.Parse(valueAsString), - MigratorDbType.UInt32 => uint.Parse(valueAsString), - MigratorDbType.UInt64 => ulong.Parse(valueAsString), - MigratorDbType.Decimal => decimal.Parse(valueAsString), - MigratorDbType.Boolean => valueAsString == "1" || valueAsString.Equals("true", StringComparison.OrdinalIgnoreCase), - MigratorDbType.String => valueAsString, - _ => throw new NotImplementedException("Type not yet supported. Please file an issue."), - }; - - filterItems.Add(filterItem); - } - } + var filterItems = IndexFilterSql.Parse(first.FilterString, + string.IsNullOrWhiteSpace(first.FilterString) ? [] : GetColumns(qualifiedTable)); var index = new Index { diff --git a/src/Migrator/Providers/Impl/Sybase/SybaseTransformationProvider.cs b/src/Migrator/Providers/Impl/Sybase/SybaseTransformationProvider.cs index 3c71d4f3..b6760ca0 100644 --- a/src/Migrator/Providers/Impl/Sybase/SybaseTransformationProvider.cs +++ b/src/Migrator/Providers/Impl/Sybase/SybaseTransformationProvider.cs @@ -191,8 +191,9 @@ public override Index[] GetIndexes(string table) public override bool IndexExists(string table, string name) => GetIndexes(table).Any(i => i.Name == name); public override string AddIndex(string table, Index index) { + ShouldApplyIndexFilters(index, supported: false); if (index.KeyColumns.Length == 0) throw new ArgumentException("An index needs key columns.", nameof(index)); - if (index.IncludeColumns.Length != 0 || index.FilterItems.Count != 0) + if (index.IncludeColumns.Length != 0) throw new NotSupportedException("ASE does not support this index's INCLUDE or filter options."); var name = index.Name ?? $"ix_{table}_{string.Join("_", index.KeyColumns)}"; ExecuteNonQuery($"CREATE {(index.Unique ? "UNIQUE " : "")}{(index.Clustered ? "CLUSTERED " : "NONCLUSTERED ")}INDEX {name} ON {table} ({string.Join(", ", index.KeyColumns)})"); diff --git a/src/Migrator/Providers/IndexFilterSql.cs b/src/Migrator/Providers/IndexFilterSql.cs new file mode 100644 index 00000000..8b2189d7 --- /dev/null +++ b/src/Migrator/Providers/IndexFilterSql.cs @@ -0,0 +1,185 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.RegularExpressions; +using DotNetProjects.Migrator.Framework; +using DotNetProjects.Migrator.Providers.Models.Indexes; +using DotNetProjects.Migrator.Providers.Models.Indexes.Enums; + +namespace DotNetProjects.Migrator.Providers; + +internal static class IndexFilterSql +{ + internal static List ParseCreateIndex(string sql, Column[] columns) + { + var where = Regex.Match(TopLevel(sql), @"\bWHERE\b", RegexOptions.IgnoreCase); + return where.Success ? Parse(sql[(where.Index + where.Length)..].TrimEnd().TrimEnd(';'), columns, sqliteIntegers: true) : []; + } + + internal static List Parse(string expression, Column[] columns, bool sqliteIntegers = false) + { + if (string.IsNullOrWhiteSpace(expression)) return []; + expression = Unwrap(expression.Trim(), stripCasts: false); + var visible = TopLevel(expression); + if (Regex.IsMatch(visible, @"\bOR\b", RegexOptions.IgnoreCase)) + throw new NotSupportedException("Index FilterItems cannot represent OR predicates: " + expression); + var and = Regex.Match(visible, @"\bAND\b", RegexOptions.IgnoreCase); + if (and.Success) + return [.. Parse(expression[..and.Index], columns, sqliteIntegers), .. Parse(expression[(and.Index + and.Length)..], columns, sqliteIntegers)]; + + if (TryParseNull(expression, out var nullFilter)) + { + nullFilter.ColumnName = FindColumn(nullFilter.ColumnName, columns).Name; + return [nullFilter]; + } + + var comparison = Regex.Match(visible, @"<>|!=|>=|<=|=|>|<"); + if (!comparison.Success) + { + // PostgreSQL may simplify boolean equality to the column or NOT column. + var negated = expression.StartsWith("NOT ", StringComparison.OrdinalIgnoreCase); + var booleanColumn = FindColumn(negated ? expression[4..] : expression, columns); + if (booleanColumn.MigratorDbType != MigratorDbType.Boolean) + throw new NotSupportedException("Unsupported index filter: " + expression); + return [new FilterItem { ColumnName = booleanColumn.Name, Filter = FilterType.EqualTo, Value = !negated }]; + } + + var column = FindColumn(expression[..comparison.Index], columns); + var literal = Unwrap(expression[(comparison.Index + comparison.Length)..].Trim()); + var type = comparison.Value switch + { + "=" => FilterType.EqualTo, "<>" or "!=" => FilterType.NotEqualTo, + ">" => FilterType.GreaterThan, ">=" => FilterType.GreaterThanOrEqualTo, + "<" => FilterType.SmallerThan, "<=" => FilterType.SmallerThanOrEqualTo, + _ => throw new NotSupportedException("Unsupported index filter: " + expression) + }; + if (literal.StartsWith("N'", StringComparison.OrdinalIgnoreCase)) literal = literal[1..]; + var quotedLiteral = Regex.IsMatch(literal, @"^'(?:[^']|'')*'$", RegexOptions.Singleline); + if (quotedLiteral) literal = literal[1..^1].Replace("''", "'"); + var culture = CultureInfo.InvariantCulture; + object value = column.MigratorDbType switch + { + // SQLite INTEGER affinity does not retain the declared CLR integer width. + MigratorDbType.Byte or MigratorDbType.SByte or MigratorDbType.Int16 or MigratorDbType.Int32 or MigratorDbType.Int64 + or MigratorDbType.UInt16 or MigratorDbType.UInt32 or MigratorDbType.UInt64 when sqliteIntegers + => long.TryParse(literal, NumberStyles.Integer, culture, out var signed) ? (object)signed : ulong.Parse(literal, culture), + MigratorDbType.String or MigratorDbType.AnsiString or MigratorDbType.StringFixedLength or MigratorDbType.AnsiStringFixedLength + when quotedLiteral => literal, + MigratorDbType.Boolean => literal.ToLowerInvariant() switch + { + "1" or "true" => true, "0" or "false" => false, + _ => throw new NotSupportedException("Unsupported boolean index filter: " + expression) + }, + MigratorDbType.Byte => byte.Parse(literal, culture), + MigratorDbType.SByte => sbyte.Parse(literal, culture), + MigratorDbType.Int16 => short.Parse(literal, culture), + MigratorDbType.Int32 => int.Parse(literal, culture), + MigratorDbType.Int64 => long.Parse(literal, culture), + MigratorDbType.UInt16 => ushort.Parse(literal, culture), + MigratorDbType.UInt32 => uint.Parse(literal, culture), + MigratorDbType.UInt64 => ulong.Parse(literal, culture), + MigratorDbType.Decimal => decimal.Parse(literal, culture), + _ => throw new NotSupportedException("Unsupported index filter column type: " + column.MigratorDbType) + }; + return [new FilterItem { ColumnName = column.Name, Filter = type, Value = value }]; + } + + private static Column FindColumn(string operand, Column[] columns) + { + var name = Unwrap(operand.Trim()); + if (name.StartsWith('[') && name.EndsWith(']')) name = name[1..^1].Replace("]]", "]"); + else if (name.StartsWith('"') && name.EndsWith('"')) name = name[1..^1].Replace("\"\"", "\""); + return columns.FirstOrDefault(c => c.Name == name) + ?? columns.FirstOrDefault(c => c.Name.Equals(name, StringComparison.OrdinalIgnoreCase)) + ?? throw new NotSupportedException("Unsupported index filter column: " + operand); + } + + private static string Unwrap(string sql, bool stripCasts = true) + { + while (true) + { + var visible = TopLevel(sql); + var cast = visible.IndexOf("::", StringComparison.Ordinal); + if (stripCasts && cast >= 0) + { + if (!Regex.IsMatch(sql[cast..], @"^::(?:text|boolean|integer|bigint|smallint|numeric|character varying|character|bpchar|varchar)(?:\(\d+(?:,\s*\d+)?\))?$", RegexOptions.IgnoreCase)) + throw new NotSupportedException("Unsupported index filter cast: " + sql); + sql = sql[..cast].Trim(); + continue; + } + if (sql.StartsWith('(') && sql.EndsWith(')') && string.IsNullOrWhiteSpace(visible)) + { sql = sql[1..^1].Trim(); continue; } + return sql; + } + } + + // Hide quoted tokens and parenthesized expressions when finding conjunctions/operators. + private static string TopLevel(string sql) + { + var result = sql.ToCharArray(); + var depth = 0; + char quote = '\0'; + for (var i = 0; i < sql.Length; i++) + { + var ch = sql[i]; + if (quote != '\0') + { + result[i] = ' '; + if (ch != quote) continue; + if (i + 1 < sql.Length && sql[i + 1] == quote) { result[++i] = ' '; continue; } + quote = '\0'; + continue; + } + if (ch is '\'' or '"' or '[') { quote = ch == '[' ? ']' : ch; result[i] = ' '; continue; } + if (ch == '(') depth++; + if (depth > 0) result[i] = ' '; + if (ch == ')') depth--; + if (depth < 0) throw new NotSupportedException("Unbalanced index filter: " + sql); + } + if (quote != '\0' || depth != 0) throw new NotSupportedException("Unbalanced index filter: " + sql); + return new string(result); + } + + internal static string Format(Dialect dialect, FilterItem filter, bool numericBooleans) + { + var column = dialect.QuoteColumnNameIfRequired(filter.ColumnName); + if (filter.Value is null or DBNull) + { + return filter.Filter switch + { + FilterType.EqualTo => $"{column} IS NULL", + FilterType.NotEqualTo => $"{column} IS NOT NULL", + _ => throw new ArgumentException("Null index filters require EqualTo or NotEqualTo.", nameof(filter)) + }; + } + + var comparison = dialect.GetComparisonStringByFilterType(filter.Filter); + var value = filter.Value switch + { + bool b => numericBooleans ? (b ? "1" : "0") : (b ? "TRUE" : "FALSE"), + string s => "'" + s.Replace("'", "''") + "'", + byte or short or int or long or sbyte or ushort or uint or ulong => Convert.ToString(filter.Value, CultureInfo.InvariantCulture), + _ => throw new NotSupportedException($"Index filters do not support values of type {filter.Value.GetType().Name}.") + }; + return $"{column} {comparison} {value}"; + } + + // Catalogs preserve different identifier quotes, but use the same NULL predicates. + internal static bool TryParseNull(string expression, out FilterItem filter) + { + filter = null; + var match = Regex.Match(expression.Trim(), @"^(?.+?)\s+IS\s+(?NOT\s+)?NULL$", RegexOptions.IgnoreCase); + if (!match.Success) return false; + var column = match.Groups["column"].Value.Trim(); + if (column.StartsWith('[') && column.EndsWith(']')) column = column[1..^1].Replace("]]", "]"); + else if (column.StartsWith('"') && column.EndsWith('"')) column = column[1..^1].Replace("\"\"", "\""); + filter = new FilterItem + { + ColumnName = column, + Filter = match.Groups["not"].Success ? FilterType.NotEqualTo : FilterType.EqualTo, + Value = null + }; + return true; + } +} diff --git a/src/Migrator/Providers/Models/Indexes/FilterItem.cs b/src/Migrator/Providers/Models/Indexes/FilterItem.cs index 6d034146..d274ed80 100644 --- a/src/Migrator/Providers/Models/Indexes/FilterItem.cs +++ b/src/Migrator/Providers/Models/Indexes/FilterItem.cs @@ -15,7 +15,8 @@ public class FilterItem public FilterType Filter { get; set; } /// - /// Gets or sets the value used in the comparison. It needs to be a static not dynamic value. Currently we support bool, byte, short, int, long + /// Gets or sets the constant comparison value: a string, boolean or integer. + /// With EqualTo or NotEqualTo, null and DBNull.Value represent IS NULL and IS NOT NULL. /// public object Value { get; set; } } \ No newline at end of file diff --git a/src/Migrator/Providers/TransformationProvider.cs b/src/Migrator/Providers/TransformationProvider.cs index e1a9ca0d..580dc2a9 100644 --- a/src/Migrator/Providers/TransformationProvider.cs +++ b/src/Migrator/Providers/TransformationProvider.cs @@ -2046,9 +2046,21 @@ public IEnumerable GetColumns(string schema, string table) return from DataRow row in tables.Rows select (row["COLUMN_NAME"] as string); } - protected void ValidateIndex(string tableName, Index index) - { - var hasFilterItems = index.FilterItems != null && index.FilterItems.Count > 0; + protected bool ShouldApplyIndexFilters(Index index, bool supported, string reason = null) + { + if (!Enum.IsDefined(index.UnsupportedFilterBehavior)) + throw new ArgumentOutOfRangeException(nameof(index.UnsupportedFilterBehavior)); + if (index.FilterItems == null || index.FilterItems.Count == 0) return false; + if (supported) return true; + if (index.UnsupportedFilterBehavior == UnsupportedIndexFilterBehavior.Ignore) return false; + throw new NotSupportedException($"{GetType().Name} does not support the requested index filters. {reason}".TrimEnd()); + } + + protected void ValidateIndex(string tableName, Index index, bool validateFilters = true) + { + if (!Enum.IsDefined(index.UnsupportedFilterBehavior)) + throw new ArgumentOutOfRangeException(nameof(index.UnsupportedFilterBehavior)); + var hasFilterItems = validateFilters && index.FilterItems != null && index.FilterItems.Count > 0; var columns = GetColumns(table: tableName); if (!TableExists(tableName)) @@ -2058,7 +2070,7 @@ protected void ValidateIndex(string tableName, Index index) foreach (var keyColumn in index.KeyColumns) { - if (!index.KeyColumns.All(x => columns.Any(y => y.Name.Equals(x, StringComparison.OrdinalIgnoreCase)))) + if (!columns.Any(column => column.Name.Equals(keyColumn, StringComparison.OrdinalIgnoreCase))) { throw new MigrationException($"Column '{keyColumn}' does not exist."); } @@ -2066,9 +2078,10 @@ protected void ValidateIndex(string tableName, Index index) if (hasFilterItems) { - if (!index.FilterItems.All(x => index.KeyColumns.Any(y => x.ColumnName.Equals(y, StringComparison.OrdinalIgnoreCase)))) + foreach (var filter in index.FilterItems) { - throw new MigrationException($"All columns in the {nameof(index.FilterItems)} should exist in the {nameof(index.KeyColumns)}."); + if (!columns.Any(column => column.Name.Equals(filter.ColumnName, StringComparison.OrdinalIgnoreCase))) + throw new MigrationException($"Filter column '{filter.ColumnName}' does not exist in table '{tableName}'."); } }