Add a program which verifies that an identity has been correctly deleted

.ci/appsettings.override.postgres.docker.json

@@ -19,7 +19,7 @@
         },
         "SqlDatabase": {
             "Provider": "Postgres",
-            "ConnectionString": "User Id=adminUi;Password=Passw0rd;Server=postgres;Port=5432;Database=enmeshed;"
+            "ConnectionString": "User ID=postgres;Password=Passw0rd;Server=postgres;Port=5432;Database=enmeshed;"
         }
     },
     "ModuleDefaults": {

.ci/appsettings.override.postgres.local.json

@@ -19,7 +19,7 @@
         },
         "SqlDatabase": {
             "Provider": "Postgres",
-            "ConnectionString": "User Id=adminUi;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;"
+            "ConnectionString": "User ID=postgres;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;"
         }
     },
     "ModuleDefaults": {

.github/workflows/test.yml

@@ -176,7 +176,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        database: [sqlserver, postgres]
+        database: [ sqlserver, postgres ]
         test-project:
           - display-name: admin-api
             path: Applications/AdminApi/test/AdminApi.Tests.Integration
@@ -207,6 +207,14 @@ jobs:
           key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj') }}
           restore-keys: ${{ runner.os }}-nuget-
 
+      - name: Install Postgres 17
+        run: |
+          sudo apt update
+          sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'  # Add Postgres APT Repo
+          curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg       # Add GPG Key
+          sudo apt update
+          sudo apt install postgresql-17
+
       - name: Download cached Docker images
         uses: actions/download-artifact@v5
         with:
@@ -236,7 +244,7 @@ jobs:
           docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database}}.yml wait admin-cli
 
       - name: Run integration tests
-        run: dotnet test --no-restore --no-build --logger "GitHubActions;summary.includeNotFoundTests=false;summary.includeSkippedTests=false;summary.includePassedTests=false" ${{matrix.test-project.path}}
+        run: dotnet test --no-restore --no-build --logger "GitHubActions;summary.includeNotFoundTests=false;summary.includeSkippedTests=false;summary.includePassedTests=true" --blame-hang-timeout 60s ${{matrix.test-project.path}}
         env:
           ADMIN_API_BASE_ADDRESS: "http://localhost:5173"
           CONSUMER_API_BASE_ADDRESS: "http://localhost:5000"
@@ -258,7 +266,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        database: [sqlserver, postgres]
+        database: [ sqlserver, postgres ]
     needs: image-test-builds
     steps:
       - name: Checkout backbone repository

Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile

@@ -89,6 +89,8 @@ FROM base AS final
 RUN apk add icu-libs
 ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=0
 
+RUN apk add postgresql17-client
+
 WORKDIR /app
 COPY --from=publish /app/publish .
 ENTRYPOINT ["dotnet", "Backbone.Job.IdentityDeletion.dll"]

Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionJobConfiguration.cs

@@ -1,5 +1,6 @@
 using System.ComponentModel.DataAnnotations;
 using Backbone.BuildingBlocks.Infrastructure.EventBus;
+using Backbone.BuildingBlocks.Infrastructure.Persistence.Database;
 
 namespace Backbone.Job.IdentityDeletion;
 
@@ -16,4 +17,7 @@ public class InfrastructureConfiguration
 {
     [Required]
     public required EventBusConfiguration EventBus { get; init; }
+
+    [Required]
+    public required DatabaseConfiguration SqlDatabase { get; init; }
 }

Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/DeletionVerifier.cs

@@ -0,0 +1,100 @@
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Exporters;
+using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Extractors;
+
+namespace Backbone.Job.IdentityDeletion.IdentityDeletionVerifier;
+
+public class DeletionVerifier : IDeletionVerifier
+{
+    private static readonly JsonSerializerOptions SERIALIZER_OPTIONS = new()
+    {
+        IndentSize = 4,
+        PropertyNameCaseInsensitive = true,
+        WriteIndented = true
+    };
+
+    private readonly IDbExporter _dbExporter;
+    private readonly ISqlExtractor _sqlExtractor;
+
+    public DeletionVerifier(IDbExporter dbExporter, ISqlExtractor sqlExtractor)
+    {
+        _dbExporter = dbExporter;
+        _sqlExtractor = sqlExtractor;
+    }
+
+    public async Task<DatabaseCheckResult> VerifyDeletion(List<string> addressesToVerify, CancellationToken cancellationToken)
+    {
+        await ExportDatabase(cancellationToken);
+
+        return await CheckExportedDatabase(addressesToVerify, cancellationToken);
+    }
+
+    private async Task ExportDatabase(CancellationToken cancellationToken)
+    {
+        if (!FilePaths.TempDirExists()) Directory.CreateDirectory(FilePaths.PATH_TO_TEMP_DIR);
+
+        await _dbExporter.ExportDb(FilePaths.PATH_TO_DUMP_FILE, cancellationToken);
+    }
+
+    private async Task<DatabaseCheckResult> CheckExportedDatabase(List<string> addressesToCheck, CancellationToken cancellationToken)
+    {
+        var result = new DatabaseCheckResult
+        {
+            Success = true,
+            FoundOccurrences = []
+        };
+
+        await foreach (var extractedTable in _sqlExtractor.ExtractTables(FilePaths.PATH_TO_DUMP_FILE).WithCancellation(cancellationToken))
+        {
+            var found = addressesToCheck.ToDictionary(i => i, _ => 0);
+
+            await foreach (var line in extractedTable.EntryLines.WithCancellation(cancellationToken))
+            {
+                foreach (var identity in addressesToCheck)
+                {
+                    var count = Regex.Matches(line, identity).Count;
+                    found[identity] += count;
+
+                    if (count != 0) result.Success = false;
+                }
+            }
+
+            if (found.Any(entry => entry.Value != 0))
+                result.FoundOccurrences.Add(extractedTable.Id, found.Where(kvp => kvp.Value != 0).ToDictionary());
+        }
+
+        return result;
+    }
+
+    public async Task SaveFoundOccurrences(DatabaseCheckResult result, CancellationToken cancellationToken)
+    {
+        var groupedOccurrences = result.FoundOccurrences
+            .GroupBy(x => x.Key.Schema)
+            .ToDictionary(
+                group => group.Key,
+                group => group.ToDictionary(
+                    v => v.Key.Table,
+                    v => v.Value
+                )
+            );
+
+        await using var stream = new FileStream(FilePaths.PATH_TO_FOUND_FILE, FileMode.Create);
+        await JsonSerializer.SerializeAsync(stream, groupedOccurrences, SERIALIZER_OPTIONS, cancellationToken);
+    }
+}
+
+public class DeletionFailedException(DatabaseCheckResult result)
+    : Exception($"Some identities were still found in the database ({result.NumberOfOccurrences} times in total). Check \"{FilePaths.PATH_TO_FOUND_FILE}\" for more details.");
+
+file static class FilePaths
+{
+    private const string DUMP_FILENAME = "db-dump.sql";
+    private const string FOUND_FILENAME = "found-identities.json";
+
+    public static readonly string PATH_TO_TEMP_DIR = Path.Combine(Path.GetTempPath(), "enmeshed", "backbone");
+    public static readonly string PATH_TO_DUMP_FILE = Path.Combine(PATH_TO_TEMP_DIR, DUMP_FILENAME);
+    public static readonly string PATH_TO_FOUND_FILE = Path.Combine(PATH_TO_TEMP_DIR, FOUND_FILENAME);
+
+    public static bool TempDirExists() => Directory.Exists(PATH_TO_TEMP_DIR);
+}

Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Exporters/IDbExporter.cs

@@ -0,0 +1,6 @@
+namespace Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Exporters;
+
+public interface IDbExporter
+{
+    Task ExportDb(string targetFile, CancellationToken cancellationToken);
+}

Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Exporters/PostgresDbExporter.cs

@@ -0,0 +1,37 @@
+using Backbone.BuildingBlocks.Infrastructure.Persistence.Database;
+using Backbone.Tooling.Extensions;
+using Microsoft.Extensions.Options;
+using PgDump;
+
+namespace Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Exporters;
+
+public class PostgresDbExporter : IDbExporter
+{
+    private readonly DatabaseConfiguration _configuration;
+    private readonly ILogger<PostgresDbExporter> _logger;
+
+    public PostgresDbExporter(IOptions<IdentityDeletionJobConfiguration> configuration, ILogger<PostgresDbExporter> logger)
+    {
+        _configuration = configuration.Value.Infrastructure.SqlDatabase;
+        _logger = logger;
+    }
+
+    public async Task ExportDb(string targetFile, CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("Starting Postgres database dump");
+
+        var parts = _configuration.ConnectionString.Split(';');
+        var host = parts.ExtractString("Server=");
+        var port = int.Parse(parts.ExtractString("Port="));
+        var username = parts.ExtractString("User ID=");
+        var password = parts.ExtractString("Password=");
+        var database = parts.ExtractString("Database=");
+
+        var connection = new ConnectionOptions(host, port, username, password, database);
+        var client = new PgClient(connection);
+
+        await client.DumpAsync(new FileOutputProvider(targetFile), 1.Minutes(), DumpFormat.Plain, cancellationToken);
+
+        _logger.LogInformation("Postgres database dump completed");
+    }
+}

Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Exporters/SqlServerDbExporter.cs

@@ -0,0 +1,66 @@
+using System.Text;
+using Backbone.BuildingBlocks.Infrastructure.Persistence.Database;
+using Microsoft.Data.SqlClient;
+using Microsoft.Extensions.Options;
+using Microsoft.SqlServer.Management.Common;
+using Microsoft.SqlServer.Management.Smo;
+
+namespace Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Exporters;
+
+public class SqlServerDbExporter : IDbExporter
+{
+    private readonly DatabaseConfiguration _configuration;
+    private readonly ILogger<SqlServerDbExporter> _logger;
+
+    public SqlServerDbExporter(IOptions<IdentityDeletionJobConfiguration> configuration, ILogger<SqlServerDbExporter> logger)
+    {
+        _configuration = configuration.Value.Infrastructure.SqlDatabase;
+        _logger = logger;
+    }
+
+    public async Task ExportDb(string targetFile, CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("Starting Sql Server database dump");
+
+        var parts = _configuration.ConnectionString.Split(';');
+        var databaseName = parts.ExtractString("Database=");
+        var server = new Server(new ServerConnection(new SqlConnection(_configuration.ConnectionString)));
+        var database = server.Databases[databaseName] ?? throw new ArgumentException($"No database with name \"{databaseName}\" found in sql server");
+        var scripter = new Scripter(server)
+        {
+            Options = new ScriptingOptions
+            {
+                EnforceScriptingOptions = true,
+                WithDependencies = false,
+                IncludeHeaders = false,
+                ScriptDrops = false,
+                AppendToFile = false,
+                ScriptSchema = false,
+                ScriptData = true,
+                IncludeIfNotExists = false,
+                Default = true,
+                Indexes = false
+            }
+        };
+
+        await using var writer = new StreamWriter(new FileStream(targetFile, FileMode.Create), Encoding.UTF8);
+        foreach (var table in database.Tables.Cast<Table>())
+        {
+            if (table.IsSystemObject) continue;
+
+            _logger.LogInformation("Writing table {schema}.{table}", table.Schema, table.Name);
+
+            await writer.WriteLineAsync($"table;{table.Schema};{table.Name}");
+
+            foreach (var line in scripter.EnumScript(new[] { table.Urn }))
+            {
+                await writer.WriteLineAsync(line);
+            }
+
+            await writer.WriteLineAsync();
+            await writer.FlushAsync(cancellationToken);
+        }
+
+        _logger.LogInformation("Sql Server database dump completed");
+    }
+}

Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Extractors/ISqlExtractor.cs

@@ -0,0 +1,18 @@
+namespace Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Extractors;
+
+public interface ISqlExtractor
+{
+    IAsyncEnumerable<ExtractedTable> ExtractTables(string file);
+}
+
+public record ExtractedTable
+{
+    public required TableId Id { get; init; }
+    public required IAsyncEnumerable<string> EntryLines { get; init; }
+}
+
+public record TableId
+{
+    public required string Schema { get; init; }
+    public required string Table { get; init; }
+}

Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Extractors/PostgresSqlExtractor.cs

@@ -0,0 +1,70 @@
+using System.Text;
+using System.Text.RegularExpressions;
+using Backbone.Tooling.Extensions;
+
+namespace Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Extractors;
+
+public partial class PostgresSqlExtractor : ISqlExtractor
+{
+    [GeneratedRegex("""
+                    \"[^\"]*\"
+                    """)]
+    private static partial Regex TableNameAndSchemaRegex();
+
+    private readonly ILogger<PostgresSqlExtractor> _logger;
+
+    public PostgresSqlExtractor(ILogger<PostgresSqlExtractor> logger)
+    {
+        _logger = logger;
+    }
+
+    public async IAsyncEnumerable<ExtractedTable> ExtractTables(string file)
+    {
+        _logger.LogInformation("Extracting dumped postgres data");
+
+        var reader = new StreamReader(new FileStream(file, FileMode.Open, FileAccess.Read), Encoding.UTF8);
+
+        while (!reader.EndOfStream)
+        {
+            var line = await reader.ReadLineAsync();
+            if (line.IsNullOrEmpty()) continue;
+
+            if (line.Trim().StartsWith("COPY"))
+            {
+                var id = ExtractTableNameAndSchema(line);
+                var entryLines = ReadCopyLines(reader);
+                yield return new ExtractedTable
+                {
+                    Id = id,
+                    EntryLines = entryLines
+                };
+            }
+        }
+
+        _logger.LogInformation("Extraction complete");
+    }
+
+    private static TableId ExtractTableNameAndSchema(string startLine)
+    {
+        var matches = TableNameAndSchemaRegex().Matches(startLine);
+        var schemaName = matches[0].Value.Trim('"');
+        var tableName = matches[1].Value.Trim('"');
+
+        return new TableId
+        {
+            Schema = schemaName,
+            Table = tableName
+        };
+    }
+
+    private static async IAsyncEnumerable<string> ReadCopyLines(StreamReader reader)
+    {
+        while (!reader.EndOfStream)
+        {
+            var line = await reader.ReadLineAsync();
+            if (line is null or "\\.") break;
+
+            yield return line;
+        }
+    }
+}