From 2f99e22bbee4fe68c138e404cc6edc53eb8d5e6c Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Mon, 18 Aug 2025 12:33:57 +0200 Subject: [PATCH 01/51] feat: Add project and launch settings --- .../IdentityDeletionVerifier.csproj | 20 +++++++++++++++++++ .../Properties/launchSettings.json | 13 ++++++++++++ Backbone.sln | 13 ++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj create mode 100644 Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Properties/launchSettings.json diff --git a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj new file mode 100644 index 0000000000..85da230e4c --- /dev/null +++ b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj @@ -0,0 +1,20 @@ + + + + Exe + net9.0 + enable + enable + + + + + + + + + + + + + diff --git a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Properties/launchSettings.json b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Properties/launchSettings.json new file mode 100644 index 0000000000..d0016ab078 --- /dev/null +++ b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Properties/launchSettings.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "profiles": { + "Init": { + "commandName": "Project", + "commandLineArgs": "init --consumerBaseUrl http://localhost:8081 --adminBaseUrl http://localhost:8082" + }, + "Check": { + "commandName": "Project", + "commandLineArgs": "check" + } + } +} diff --git a/Backbone.sln b/Backbone.sln index 3b6b12209d..7a09ad5e71 100644 --- a/Backbone.sln +++ b/Backbone.sln @@ -396,6 +396,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BuildingBlocks.Module", "Bu EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Announcements.Domain.Tests", "Modules\Announcements\test\Announcements.Domain.Tests\Announcements.Domain.Tests.csproj", "{44300266-A7BD-4D04-A719-6EAAEFEB1F95}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "IdentityDeletionVerifier", "IdentityDeletionVerifier", "{584BC088-D517-4799-80DF-73078E7BAC2F}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{E8D1B480-AE46-4DE0-BD74-EDC70E95B44C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "IdentityDeletionVerifier", "Applications\IdentityDeletionVerifier\src\IdentityDeletionVerifier\IdentityDeletionVerifier.csproj", "{DC3E495C-60E8-4885-86D6-73466893CBB4}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -878,6 +884,10 @@ Global {44300266-A7BD-4D04-A719-6EAAEFEB1F95}.Debug|Any CPU.Build.0 = Debug|Any CPU {44300266-A7BD-4D04-A719-6EAAEFEB1F95}.Release|Any CPU.ActiveCfg = Release|Any CPU {44300266-A7BD-4D04-A719-6EAAEFEB1F95}.Release|Any CPU.Build.0 = Release|Any CPU + {DC3E495C-60E8-4885-86D6-73466893CBB4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DC3E495C-60E8-4885-86D6-73466893CBB4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DC3E495C-60E8-4885-86D6-73466893CBB4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DC3E495C-60E8-4885-86D6-73466893CBB4}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1061,6 +1071,9 @@ Global {74CDB906-8BC9-42F7-A3FA-2B675A240D51} = {399BA0C2-C130-4C8E-8F2D-8BB45AB9FD1A} {0462B097-733E-4CC0-945D-664AECCA04C3} = {06D714AE-EDF4-421C-9340-EDA6FCDF491F} {44300266-A7BD-4D04-A719-6EAAEFEB1F95} = {399BA0C2-C130-4C8E-8F2D-8BB45AB9FD1A} + {584BC088-D517-4799-80DF-73078E7BAC2F} = {44C9D62D-813D-497A-8DDF-C06E515CB22E} + {E8D1B480-AE46-4DE0-BD74-EDC70E95B44C} = {584BC088-D517-4799-80DF-73078E7BAC2F} + {DC3E495C-60E8-4885-86D6-73466893CBB4} = {E8D1B480-AE46-4DE0-BD74-EDC70E95B44C} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {1F3BD2C6-7CB3-450F-A21A-23EA520D5B7A} From 6dee93a0712cee11c60d15269c6d178fbb9d07ec Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Mon, 18 Aug 2025 12:34:20 +0200 Subject: [PATCH 02/51] feat: Add extensions for the console and static file paths --- .../Extensions/AnsiConsoleExtensions.cs | 16 ++++++++++++++++ .../src/IdentityDeletionVerifier/FilePaths.cs | 14 ++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Extensions/AnsiConsoleExtensions.cs create mode 100644 Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/FilePaths.cs diff --git a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Extensions/AnsiConsoleExtensions.cs b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Extensions/AnsiConsoleExtensions.cs new file mode 100644 index 0000000000..5f3abeb21f --- /dev/null +++ b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Extensions/AnsiConsoleExtensions.cs @@ -0,0 +1,16 @@ +using Backbone.BuildingBlocks.SDK.Endpoints.Common.Types; +using Backbone.Tooling.Extensions; +using Spectre.Console; + +namespace Backbone.IdentityDeletionVerifier.Extensions; + +public static class AnsiConsoleExtensions +{ + public static bool WriteResult(this IAnsiConsole console, params IResponse[] responses) + { + var success = responses.All(r => r.IsSuccess); + console.WriteLine(success ? Emoji.Known.CheckMarkButton : Emoji.Known.CrossMark); + + return success; + } +} diff --git a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/FilePaths.cs b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/FilePaths.cs new file mode 100644 index 0000000000..25cd01e83c --- /dev/null +++ b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/FilePaths.cs @@ -0,0 +1,14 @@ +using System.Text.RegularExpressions; + +namespace Backbone.IdentityDeletionVerifier; + +public static partial class FilePaths +{ + public static readonly string PATH_TO_TEMP_DIR = Path.Combine(Path.GetTempPath(), "enmeshed", "backbone"); + public const string IDENTITIES_FILENAME = "deleted-identities.txt"; + public static readonly Regex EXPORT_FILE_PATTERN = MyRegex(); + public static readonly string PATH_TO_IDENTITIES_FILE = Path.Combine(PATH_TO_TEMP_DIR, IDENTITIES_FILENAME); + + [GeneratedRegex(@"export-\d{8}_\d{6}\.zip$")] + private static partial Regex MyRegex(); +} From 1eae5602c798a43d3fedbad976783957d7e94b9b Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Mon, 18 Aug 2025 12:34:44 +0200 Subject: [PATCH 03/51] feat: Add the two commands and the main program --- .../Commands/CheckCommand.cs | 117 +++++++ .../Commands/InitCommand.cs | 287 ++++++++++++++++++ .../src/IdentityDeletionVerifier/Program.cs | 15 + 3 files changed, 419 insertions(+) create mode 100644 Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/CheckCommand.cs create mode 100644 Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/InitCommand.cs create mode 100644 Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Program.cs diff --git a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/CheckCommand.cs b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/CheckCommand.cs new file mode 100644 index 0000000000..56328b5982 --- /dev/null +++ b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/CheckCommand.cs @@ -0,0 +1,117 @@ +using System.CommandLine; +using System.IO.Compression; +using System.Text; +using Spectre.Console; + +namespace Backbone.IdentityDeletionVerifier.Commands; + +public class CheckCommand : Command +{ + public CheckCommand() : base("check", "Check the exported database file for the given identity address in the temp directory") + { + SetAction(Handle); + } + + private async Task Handle(ParseResult _, CancellationToken cancellationToken) + { + if (!DirectoryExists()) + { + AnsiConsole.MarkupLineInterpolated($"[red bold]The temp directory[/][grey bold]{FilePaths.PATH_TO_TEMP_DIR} [/][red bold]doesn't exist.[/]"); + return -1; + } + + if (!IdentitiesFileExists()) + { + AnsiConsole.MarkupLine("[red bold]The deleted identities file doesn't exist. Run the Init command first.[/]"); + return -1; + } + + if (!ExportFileExists()) + { + AnsiConsole.MarkupLine("[red bold]No exported database file found. Run the Admin Cli Export Database command first.[/]"); + return -1; + } + + var a = await GetIdentityToCheck(); + if (a == null) + { + AnsiConsole.MarkupLineInterpolated($"[red bold]The identities file couldn't be read or has no Identity[/]"); + return -1; + } + + AnsiConsole.MarkupLineInterpolated($"[green bold]Identity to check:[/] [grey bold]{a}[/]"); + + return await CheckExportFileForIdentities(GetLatestExportFile(), a); + } + + private bool DirectoryExists() => Directory.Exists(FilePaths.PATH_TO_TEMP_DIR); + private bool IdentitiesFileExists() => File.Exists(FilePaths.PATH_TO_IDENTITIES_FILE); + private bool ExportFileExists() => Directory.EnumerateFiles(FilePaths.PATH_TO_TEMP_DIR).Any(FilePaths.EXPORT_FILE_PATTERN.IsMatch); + private string GetLatestExportFile() => Directory.EnumerateFiles(FilePaths.PATH_TO_TEMP_DIR).Where(e => FilePaths.EXPORT_FILE_PATTERN.IsMatch(e)).Max()!; + + private async Task GetIdentityToCheck() + { + using var reader = new StreamReader(File.OpenRead(FilePaths.PATH_TO_IDENTITIES_FILE), Encoding.UTF8); + + return await reader.ReadLineAsync(); + } + + private async Task CheckExportFileForIdentities(string exportFile, string identityToCheck) + { + return await AnsiConsole.Progress() + .AutoClear(false) + .HideCompleted(false) + .Columns( + new TaskDescriptionColumn(), + new ProgressBarColumn(), + new PercentageColumn(), + new RemainingTimeColumn(), + new SpinnerColumn(Spinner.Known.Clock) + ) + .StartAsync(async ctx => + { + using var archive = ZipFile.OpenRead(exportFile); + + var found = 0; + var tasks = archive.Entries + .Select(e => ctx.AddTask(e.Name, autoStart: false)) + .ToList(); + + foreach (var (index, entry) in archive.Entries.Index()) + { + var task = tasks[index]; + task.StartTask(); + found += await CheckCsvFileForIdentity(entry, identityToCheck, task); + } + + return found; + }); + } + + private async Task CheckCsvFileForIdentity(ZipArchiveEntry file, string identityToCheck, ProgressTask progressReporter) + { + using var reader = new StreamReader(file.Open(), Encoding.UTF8); + List lines = []; + var found = 0; + + while (!reader.EndOfStream) + lines.Add(await reader.ReadLineAsync() ?? string.Empty); + + progressReporter.MaxValue = lines.Count; + + foreach (var line in lines) + { + var count = line + .Split(',') + .Count(s => string.Equals(s, identityToCheck, StringComparison.OrdinalIgnoreCase)); + + if (count != 0) + AnsiConsole.MarkupLineInterpolated($"[red bold]Found[/] [grey bold]{count}[/] [red bold]occurrences in[/] [grey bold]{file.Name}[/][red bold]:[/] [grey bold]{line}[/]"); + + found += count; + progressReporter.Increment(1); + } + + return found; + } +} diff --git a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/InitCommand.cs b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/InitCommand.cs new file mode 100644 index 0000000000..28bfc798d1 --- /dev/null +++ b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/InitCommand.cs @@ -0,0 +1,287 @@ +using System.CommandLine; +using System.Net; +using System.Text; +using System.Text.Unicode; +using Backbone.AdminApi.Sdk.Endpoints.Identities.Types.Requests; +using Backbone.BuildingBlocks.SDK.Endpoints.Common.Types; +using Backbone.ConsumerApi.Sdk; +using Backbone.ConsumerApi.Sdk.Authentication; +using Backbone.ConsumerApi.Sdk.Endpoints.Files.Types.Requests; +using Backbone.ConsumerApi.Sdk.Endpoints.Identities.Types.Requests; +using Backbone.ConsumerApi.Sdk.Endpoints.Messages.Types.Requests; +using Backbone.ConsumerApi.Sdk.Endpoints.PushNotifications.Types.Requests; +using Backbone.ConsumerApi.Sdk.Endpoints.Relationships.Types; +using Backbone.ConsumerApi.Sdk.Endpoints.Relationships.Types.Requests; +using Backbone.ConsumerApi.Sdk.Endpoints.RelationshipTemplates.Types.Requests; +using Backbone.ConsumerApi.Sdk.Endpoints.SyncRuns.Types.Requests; +using Backbone.ConsumerApi.Sdk.Endpoints.Tokens.Types; +using Backbone.ConsumerApi.Sdk.Endpoints.Tokens.Types.Requests; +using Backbone.Crypto; +using Backbone.IdentityDeletionVerifier.Extensions; +using Backbone.Tooling; +using Backbone.Tooling.Extensions; +using Spectre.Console; +using AdminClient = Backbone.AdminApi.Sdk.Client; + +namespace Backbone.IdentityDeletionVerifier.Commands; + +public class InitCommand : Command +{ + private static readonly ClientCredentials CREDENTIALS = new("test", "test"); + private const string PASSWORD = "password"; + private const string ADMIN_API_KEY = "test"; + private const string DUMMY_STRING = "AAAA"; + private static readonly byte[] DUMMY_DATA = DUMMY_STRING.GetBytes(); + + public InitCommand() : base("init", "Creates an identity with relationships, messages etc. and starts an immediate deletion process") + { + var consumerBaseUrl = new Option("--consumerBaseUrl") + { + Required = true, + Description = "The base url for the Consumer Api" + }; + + var adminBaseUrl = new Option("--adminBaseUrl") + { + Required = true, + Description = "The base url for the Admin Api" + }; + + Options.Add(consumerBaseUrl); + Options.Add(adminBaseUrl); + SetAction((result, _) => Handle(result.GetRequiredValue(consumerBaseUrl), result.GetRequiredValue(adminBaseUrl))); + } + + private async Task Handle(string consumerBaseUrl, string adminBaseUrl) + { + List results = []; + + var ex = await AnsiConsole.Status() + .Spinner(Spinner.Known.Clock) + .StartAsync("Preparing the identity deletion", async _ => + { + try + { + var (a, b, admin) = await CreateClients(consumerBaseUrl, adminBaseUrl); + results.Add(await CreateChallenge(a)); + results.Add(await CreateFile(a)); + results.Add(await CreatePushNotificationHandle(a)); + results.Add(await CreateIndividualQuota(a, admin)); + results.Add(await CreateTokens(a, b)); + results.Add(await CreateDatawallet(a)); + results.Add(await StartSyncRun(a)); + results.Add(await CreateRelationship(a, b)); + results.Add(await SendMessage(a, b)); + + results.Add(await StartDeletionProcesses(a, b)); + + await WriteIdentitiesToFile(a, b); + } + catch (Exception e) + { + AnsiConsole.WriteException(e); + return false; + } + + return true; + }); + results.Add(ex); + + return results.Contains(false) ? 1 : 0; + } + + private async Task<(Client, Client, AdminClient)> CreateClients(string consumerBaseUrl, string adminBaseUrl) + { + AnsiConsole.WriteLine("Creating clients..."); + var a = await Client.CreateForNewIdentity(consumerBaseUrl, CREDENTIALS, PASSWORD); + var b = await Client.CreateForNewIdentity(consumerBaseUrl, CREDENTIALS, PASSWORD); + var admin = AdminClient.Create(adminBaseUrl, ADMIN_API_KEY); + + AnsiConsole.MarkupLineInterpolated($"A: [green bold]{a.IdentityData?.Address}[/]"); + AnsiConsole.MarkupLineInterpolated($"B: [green bold]{b.IdentityData?.Address}[/]"); + AnsiConsole.WriteLine(Emoji.Known.CheckMarkButton); + + return (a, b, admin); + } + + private async Task CreateChallenge(Client a) + { + AnsiConsole.WriteLine("Creating challenge..."); + var response = await a.Challenges.CreateChallenge(); + return AnsiConsole.Console.WriteResult(response); + } + + private async Task CreateFile(Client a) + { + AnsiConsole.WriteLine("Creating file..."); + var response = await a.Files.UploadFile(new CreateFileRequest + { + Content = new MemoryStream("Content".GetBytes()), + Owner = a.IdentityData!.Address, + OwnerSignature = DUMMY_STRING, + CipherHash = DUMMY_STRING, + ExpiresAt = DateTime.UtcNow.AddDays(1), + EncryptedProperties = DUMMY_STRING + }); + return AnsiConsole.Console.WriteResult(response); + } + + private async Task CreatePushNotificationHandle(Client a) + { + AnsiConsole.WriteLine("Creating push notification handle..."); + var response = await a.PushNotifications.RegisterForPushNotifications(new UpdateDeviceRegistrationRequest + { + AppId = "de.bildungsraum.wallet.experimental", + Handle = "asdsdaasdasdasds", + Platform = "dummy" + }); + return AnsiConsole.Console.WriteResult(response); + } + + private async Task CreateIndividualQuota(Client a, AdminClient admin) + { + AnsiConsole.WriteLine("Creating individual quota..."); + var response = await admin.Identities.CreateIndividualQuota(a.IdentityData!.Address, new CreateQuotaForIdentityRequest + { + Max = 100, + MetricKey = "NumberOfSentMessages", + Period = "hour" + }); + return AnsiConsole.Console.WriteResult(response); + } + + private async Task CreateTokens(Client a, Client b) + { + AnsiConsole.WriteLine("Creating tokens (one from A, one for A, one allocated by A)..."); + var tA = await a.Tokens.CreateToken(new CreateTokenRequest + { + Content = DUMMY_DATA, + ExpiresAt = DateTime.UtcNow.AddDays(1), + ForIdentity = null, + Password = null + }); + var tForA = await b.Tokens.CreateToken(new CreateTokenRequest + { + Content = DUMMY_DATA, + ExpiresAt = DateTime.UtcNow.AddDays(1), + ForIdentity = a.IdentityData!.Address, + Password = null + }); + var tAllocatedByA = await b.Tokens.CreateToken(new CreateTokenRequest + { + Content = DUMMY_DATA, + ExpiresAt = DateTime.UtcNow.AddDays(1), + ForIdentity = a.IdentityData!.Address, + Password = null + }); + + //var allocationResponse = tAllocatedByA.IsSuccess ? await a.Tokens.GetToken(tAllocatedByA.Result!.Id) : DummyErrorResponse(); + + return AnsiConsole.Console.WriteResult(tA, tForA, tAllocatedByA /*, allocationResponse*/); + } + + private async Task CreateDatawallet(Client a) + { + AnsiConsole.WriteLine("Creating datawallet..."); + AnsiConsole.WriteLine("Not yet implemented"); + + //TODO: Timo (How to create a datawallet?) + await Task.CompletedTask; + + return true; + } + + private async Task StartSyncRun(Client a) + { + AnsiConsole.WriteLine("Starting sync run..."); + var response = await a.SyncRuns.StartSyncRun(new StartSyncRunRequest + { + Duration = null, + Type = SyncRunType.ExternalEventSync + }, 1); + return AnsiConsole.Console.WriteResult(response); + } + + private async Task CreateRelationship(Client a, Client b) + { + AnsiConsole.WriteLine("Creating relationship..."); + var templateForA = await b.RelationshipTemplates.CreateTemplate(new CreateRelationshipTemplateRequest { Content = DUMMY_DATA, ForIdentity = a.IdentityData!.Address }); + if (templateForA.IsError) return false; + + var templateByA = await a.RelationshipTemplates.CreateTemplate(new CreateRelationshipTemplateRequest { Content = DUMMY_DATA }); + + var templateAllocatedByA = await b.RelationshipTemplates.CreateTemplate(new CreateRelationshipTemplateRequest { Content = DUMMY_DATA }); + if (templateAllocatedByA.IsError) return false; + var getTemplateResponse = await a.RelationshipTemplates.GetTemplate(templateAllocatedByA.Result!.Id); + + var createRelationshipResponse = await a.Relationships.CreateRelationship(new CreateRelationshipRequest { Content = DUMMY_DATA, RelationshipTemplateId = templateForA.Result!.Id }); + if (createRelationshipResponse.IsError) return false; + + var acceptRelationshipResponse = await b.Relationships.AcceptRelationship(createRelationshipResponse.Result!.Id, new AcceptRelationshipRequest { CreationResponseContent = DUMMY_DATA }); + + return AnsiConsole.Console.WriteResult(templateForA, templateByA, templateAllocatedByA, getTemplateResponse, createRelationshipResponse, acceptRelationshipResponse); + } + + private async Task SendMessage(Client a, Client b) + { + AnsiConsole.WriteLine("Sending message from A to B..."); + var sendMessageResponse = await a.Messages.SendMessage(new SendMessageRequest + { + Attachments = [], + Body = "Message".GetBytes(), + Recipients = + [ + new SendMessageRequestRecipientInformation + { + Address = b.IdentityData!.Address, + EncryptedKey = ConvertibleString.FromUtf8("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA").BytesRepresentation + } + ] + }); + return AnsiConsole.Console.WriteResult(sendMessageResponse); + } + + private async Task StartDeletionProcesses(Client a, Client b) + { + AnsiConsole.WriteLine("Starting deletion processes..."); + + var responseA = await a.Identities.StartDeletionProcess(new StartDeletionProcessRequest { LengthOfGracePeriodInDays = 0 }); + var responseB = await b.Identities.StartDeletionProcess(new StartDeletionProcessRequest { LengthOfGracePeriodInDays = 0.1 }); + //TODO Timo: Until the Identity Deletion Bug PR is merged, A and B can't be deleted at the same time (therefore the short grace period) + + return AnsiConsole.Console.WriteResult(responseA, responseB); + } + + private async Task WriteIdentitiesToFile(Client a, Client b) + { + AnsiConsole.WriteLine("Writing identity addresses to a temp file..."); + + var file = new StreamWriter(new FileStream(FilePaths.PATH_TO_IDENTITIES_FILE, FileMode.Create), Encoding.UTF8); + + await file.WriteLineAsync(a.IdentityData!.Address); + await file.WriteLineAsync(b.IdentityData!.Address); + + await file.FlushAsync(); + file.Close(); + + AnsiConsole.WriteLine(Emoji.Known.CheckMarkButton); + } + + private static ApiResponse DummyErrorResponse() + { + return new ApiResponse + { + ContentType = null, + Error = new ApiError + { + Code = "error.dummy", + Data = null, + Message = "An error occured", + Docs = "", + Id = "", + Time = DateTime.UtcNow + }, + Status = HttpStatusCode.BadRequest + }; + } +} diff --git a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Program.cs b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Program.cs new file mode 100644 index 0000000000..495c0b142c --- /dev/null +++ b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Program.cs @@ -0,0 +1,15 @@ +// See https://aka.ms/new-console-template for more information + +using System.CommandLine; +using System.Text; +using Backbone.IdentityDeletionVerifier.Commands; + +Console.OutputEncoding = Encoding.UTF8; + +var command = new RootCommand +{ + new InitCommand(), + new CheckCommand() +}; + +return await command.Parse(args).InvokeAsync(); From 0aeba1bb9d355303602f41224ae245ab12e38e5a Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 19 Aug 2025 09:45:30 +0200 Subject: [PATCH 04/51] feat: Prepare github workflow --- .github/workflows/test.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a27af0d001..c7fb62b985 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -173,7 +173,7 @@ jobs: strategy: fail-fast: false matrix: - database: [sqlserver, postgres] + database: [ sqlserver, postgres ] test-project: - display-name: admin-api path: Applications/AdminApi/test/AdminApi.Tests.Integration @@ -255,7 +255,7 @@ jobs: strategy: fail-fast: false matrix: - database: [sqlserver, postgres] + database: [ sqlserver, postgres ] needs: image-test-builds steps: - name: Checkout backbone repository @@ -357,3 +357,20 @@ jobs: run: dotnet restore /p:ContinuousIntegrationBuild=true ./Backbone.sln - name: Validate Licenses run: nuget-license --input ./Backbone.sln --allowed-license-types ./.ci/allowedLicenses.json --ignored-packages ./.ci/ignoredPackages.json --output table --error-only + + verify-identity-deletion: + name: Validate Identity deletion (on ${{matrix.database.type}}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + database: + - type: postgres + env: + Database__Provider: Postgres + Database__ConnectionString: "Server=postgres;Database=enmeshed;User Id=devices;Password=Passw0rd;Port=5432" + - type: sqlserver + env: + Database__Provider: SqlServer + Database__ConnectionString: "Server=sqlserver;Database=enmeshed;User Id=devices;Password=Passw0rd;TrustServerCertificate=True" + needs: image-test-builds From fd4809577e445f81a1473b4ef1a98e4948d9888b Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Wed, 20 Aug 2025 11:58:34 +0200 Subject: [PATCH 05/51] ci: Finish workflow --- .github/workflows/test.yml | 62 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c7fb62b985..40dbd7fec3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -374,3 +374,65 @@ jobs: Database__Provider: SqlServer Database__ConnectionString: "Server=sqlserver;Database=enmeshed;User Id=devices;Password=Passw0rd;TrustServerCertificate=True" needs: image-test-builds + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Setup Dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 9.x + + - name: Cache NuGet packages + uses: actions/cache@v4 + with: + path: ~/.nuget/packages + key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj') }} + restore-keys: ${{ runner.os }}-nuget- + + - name: Download cached Docker images + uses: actions/download-artifact@v5 + with: + path: /tmp + pattern: docker-* + merge-multiple: true + + - name: Load Docker images and build applications + run: | + { + ./.ci/loadDockerImages.sh + } & + { + # The following two lines are for the identity deletion job only + mv appsettings.override.json appsettings.override.json.bak + cp .ci/appsettings.override.${{matrix.database.type}}.local.json appsettings.override.json + + dotnet restore ./Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj + dotnet restore ./Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Job.IdentityDeletion.csproj + dotnet restore ./Applications/AdminCli/src/AdminCli/AdminCli.csproj + + dotnet build --no-restore ./Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj + dotnet build --no-restore ./Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Job.IdentityDeletion.csproj + dotnet build --no-restore ./Applications/AdminCli/src/AdminCli/AdminCli.csproj + } + wait + + - name: Start compose stack + run: | + docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database.type}}.yml down -v + docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database.type}}.yml up --no-build --wait -d + docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database.type}}.yml wait admin-cli + + - name: Create identities, relationships etc. and initiate deletion processes + run: dotnet run --no-build --no-restore --project ./Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj init --consumerBaseUrl http://localhost:8081 --adminBaseUrl http://localhost:8082 + + - name: Run Identity deletion job + run: dotnet run --no-build --no-restore --project ./Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Job.IdentityDeletion.csproj --Worker ActualDeletionWorker + + - name: Export databse via Admin Cli + run: dotnet run --no-build --no-restore --project ./Applications/AdminCli/src/AdminCli/AdminCli.csproj database export --sensitive + env: ${{matrix.database.env}} + + - name: Check exported database file + run: dotnet run --no-build --no-restore --project ./Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj check + From bc740733e26e0c8f52718a76d9ce37e7a81c9358 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Wed, 20 Aug 2025 12:09:55 +0200 Subject: [PATCH 06/51] fix: Change ports of baseUrls --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 40dbd7fec3..ab03570051 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -424,7 +424,7 @@ jobs: docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database.type}}.yml wait admin-cli - name: Create identities, relationships etc. and initiate deletion processes - run: dotnet run --no-build --no-restore --project ./Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj init --consumerBaseUrl http://localhost:8081 --adminBaseUrl http://localhost:8082 + run: dotnet run --no-build --no-restore --project ./Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj init --consumerBaseUrl http://localhost:5000 --adminBaseUrl http://localhost:5173 - name: Run Identity deletion job run: dotnet run --no-build --no-restore --project ./Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Job.IdentityDeletion.csproj --Worker ActualDeletionWorker From d00bc697b83e3c9cbfb98a1e9b0ab67a6bf3b8fe Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Wed, 20 Aug 2025 12:26:08 +0200 Subject: [PATCH 07/51] fix: Create temp export directory if it doesn't exist --- .../src/IdentityDeletionVerifier/Commands/InitCommand.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/InitCommand.cs b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/InitCommand.cs index 28bfc798d1..2d2d68f436 100644 --- a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/InitCommand.cs +++ b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/InitCommand.cs @@ -256,6 +256,9 @@ private async Task WriteIdentitiesToFile(Client a, Client b) { AnsiConsole.WriteLine("Writing identity addresses to a temp file..."); + if (!Directory.Exists(FilePaths.PATH_TO_TEMP_DIR)) + Directory.CreateDirectory(FilePaths.PATH_TO_TEMP_DIR); + var file = new StreamWriter(new FileStream(FilePaths.PATH_TO_IDENTITIES_FILE, FileMode.Create), Encoding.UTF8); await file.WriteLineAsync(a.IdentityData!.Address); From 93020f90dfe06e7715a4af39131e674eca9325a1 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Wed, 20 Aug 2025 12:26:59 +0200 Subject: [PATCH 08/51] chore: Add failure archive of docker logs --- .github/workflows/test.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ab03570051..2e39f1dc96 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -435,4 +435,14 @@ jobs: - name: Check exported database file run: dotnet run --no-build --no-restore --project ./Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj check - + + - name: Save Docker Logs + if: failure() + run: docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database}}.yml logs > logs.txt + + - name: Archive logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: ${{matrix.test-project.display-name}}-${{matrix.database}} + path: logs.txt From 65592cc1c55152c16cfed232f2c431ef4ce1d430 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Wed, 20 Aug 2025 12:50:38 +0200 Subject: [PATCH 09/51] ci: upload exported db file and identity file (not only on failure for now) --- .github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e39f1dc96..55e1bd6e45 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -446,3 +446,9 @@ jobs: with: name: ${{matrix.test-project.display-name}}-${{matrix.database}} path: logs.txt + + - name: Archive exported database and identity files + uses: actions/upload-artifact@v4 + with: + name: exported-db + path: /tmp/enmeshed/backbone From 0087b7ae683b56b6dc5445a0b026146d7c872d01 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Wed, 20 Aug 2025 13:04:18 +0200 Subject: [PATCH 10/51] fix: Fix variables --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 55e1bd6e45..048faa11dc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -444,11 +444,11 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: ${{matrix.test-project.display-name}}-${{matrix.database}} + name: identity-deletion-verifier-docker-${{matrix.database.type}} path: logs.txt - name: Archive exported database and identity files uses: actions/upload-artifact@v4 with: - name: exported-db + name: identity-deletion-verifier-exported-db-${{matrix.database.type}} path: /tmp/enmeshed/backbone From f9692c96a312d57c31cdc088cd44e34aaa07ca15 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Wed, 20 Aug 2025 13:12:13 +0200 Subject: [PATCH 11/51] chore: Only upload exported db files on failure --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 048faa11dc..5d5a666374 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -438,7 +438,7 @@ jobs: - name: Save Docker Logs if: failure() - run: docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database}}.yml logs > logs.txt + run: docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database.type}}.yml logs > logs.txt - name: Archive logs if: failure() @@ -448,6 +448,7 @@ jobs: path: logs.txt - name: Archive exported database and identity files + if: failure() uses: actions/upload-artifact@v4 with: name: identity-deletion-verifier-exported-db-${{matrix.database.type}} From e3f202137135e9b01ed71dd45f0d37b054a44dc4 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Fri, 22 Aug 2025 13:11:25 +0200 Subject: [PATCH 12/51] fix: Delete token allocations instead of anonymizing them when deleting the allocating identity --- .../Identities/IdentityDeleter.cs | 4 +-- ...nymizeTokenAllocationsOfIdentityCommand.cs | 8 ----- .../Handler.cs | 29 ------------------- .../Validator.cs | 14 --------- ...DeleteTokenAllocationsOfIdentityCommand.cs | 8 +++++ .../Handler.cs | 26 +++++++++++++++++ .../Validator.cs | 14 +++++++++ .../src/Tokens.Domain/Entities/Token.cs | 6 ++-- 8 files changed, 52 insertions(+), 57 deletions(-) delete mode 100644 Modules/Tokens/src/Tokens.Application/Tokens/Commands/AnonymizeTokenAllocationsOfIdentity/AnonymizeTokenAllocationsOfIdentityCommand.cs delete mode 100644 Modules/Tokens/src/Tokens.Application/Tokens/Commands/AnonymizeTokenAllocationsOfIdentity/Handler.cs delete mode 100644 Modules/Tokens/src/Tokens.Application/Tokens/Commands/AnonymizeTokenAllocationsOfIdentity/Validator.cs create mode 100644 Modules/Tokens/src/Tokens.Application/Tokens/Commands/DeleteTokenAllocationsOfIdentity/DeleteTokenAllocationsOfIdentityCommand.cs create mode 100644 Modules/Tokens/src/Tokens.Application/Tokens/Commands/DeleteTokenAllocationsOfIdentity/Handler.cs create mode 100644 Modules/Tokens/src/Tokens.Application/Tokens/Commands/DeleteTokenAllocationsOfIdentity/Validator.cs diff --git a/Modules/Tokens/src/Tokens.Application/Identities/IdentityDeleter.cs b/Modules/Tokens/src/Tokens.Application/Identities/IdentityDeleter.cs index 82a2e7c3ae..e21956180c 100644 --- a/Modules/Tokens/src/Tokens.Application/Identities/IdentityDeleter.cs +++ b/Modules/Tokens/src/Tokens.Application/Identities/IdentityDeleter.cs @@ -1,7 +1,7 @@ using Backbone.BuildingBlocks.Application.Identities; using Backbone.DevelopmentKit.Identity.ValueObjects; -using Backbone.Modules.Tokens.Application.Tokens.Commands.AnonymizeTokenAllocationsOfIdentity; using Backbone.Modules.Tokens.Application.Tokens.Commands.AnonymizeTokensForIdentity; +using Backbone.Modules.Tokens.Application.Tokens.Commands.DeleteTokenAllocationsOfIdentity; using Backbone.Modules.Tokens.Application.Tokens.Commands.DeleteTokensOfIdentity; using MediatR; @@ -22,7 +22,7 @@ public async Task Delete(IdentityAddress identityAddress) { await _mediator.Send(new DeleteTokensOfIdentityCommand { IdentityAddress = identityAddress }); await _mediator.Send(new AnonymizeTokensForIdentityCommand { IdentityAddress = identityAddress }); - await _mediator.Send(new AnonymizeTokenAllocationsOfIdentityCommand { IdentityAddress = identityAddress }); + await _mediator.Send(new DeleteTokenAllocationsOfIdentityCommand { IdentityAddress = identityAddress }); await _deletionProcessLogger.LogDeletion(identityAddress, "Tokens"); } } diff --git a/Modules/Tokens/src/Tokens.Application/Tokens/Commands/AnonymizeTokenAllocationsOfIdentity/AnonymizeTokenAllocationsOfIdentityCommand.cs b/Modules/Tokens/src/Tokens.Application/Tokens/Commands/AnonymizeTokenAllocationsOfIdentity/AnonymizeTokenAllocationsOfIdentityCommand.cs deleted file mode 100644 index 86a55247e9..0000000000 --- a/Modules/Tokens/src/Tokens.Application/Tokens/Commands/AnonymizeTokenAllocationsOfIdentity/AnonymizeTokenAllocationsOfIdentityCommand.cs +++ /dev/null @@ -1,8 +0,0 @@ -using MediatR; - -namespace Backbone.Modules.Tokens.Application.Tokens.Commands.AnonymizeTokenAllocationsOfIdentity; - -public class AnonymizeTokenAllocationsOfIdentityCommand : IRequest -{ - public required string IdentityAddress { get; init; } -} diff --git a/Modules/Tokens/src/Tokens.Application/Tokens/Commands/AnonymizeTokenAllocationsOfIdentity/Handler.cs b/Modules/Tokens/src/Tokens.Application/Tokens/Commands/AnonymizeTokenAllocationsOfIdentity/Handler.cs deleted file mode 100644 index 42509ccf74..0000000000 --- a/Modules/Tokens/src/Tokens.Application/Tokens/Commands/AnonymizeTokenAllocationsOfIdentity/Handler.cs +++ /dev/null @@ -1,29 +0,0 @@ -using Backbone.DevelopmentKit.Identity.ValueObjects; -using Backbone.Modules.Tokens.Application.Infrastructure.Persistence.Repository; -using Backbone.Modules.Tokens.Domain.Entities; -using MediatR; -using Microsoft.Extensions.Options; - -namespace Backbone.Modules.Tokens.Application.Tokens.Commands.AnonymizeTokenAllocationsOfIdentity; - -public class Handler : IRequestHandler -{ - private readonly ITokensRepository _tokensRepository; - private readonly ApplicationConfiguration _applicationConfiguration; - - public Handler(ITokensRepository tokensRepository, IOptions applicationOptions) - { - _tokensRepository = tokensRepository; - _applicationConfiguration = applicationOptions.Value; - } - - public async Task Handle(AnonymizeTokenAllocationsOfIdentityCommand request, CancellationToken cancellationToken) - { - var tokens = (await _tokensRepository.ListWithoutContent(Token.HasAllocationFor(IdentityAddress.Parse(request.IdentityAddress)), cancellationToken, track: true)).ToList(); - - foreach (var token in tokens) - token.AnonymizeTokenAllocation(request.IdentityAddress, _applicationConfiguration.DidDomainName); - - await _tokensRepository.Update(tokens, cancellationToken); - } -} diff --git a/Modules/Tokens/src/Tokens.Application/Tokens/Commands/AnonymizeTokenAllocationsOfIdentity/Validator.cs b/Modules/Tokens/src/Tokens.Application/Tokens/Commands/AnonymizeTokenAllocationsOfIdentity/Validator.cs deleted file mode 100644 index 8771f07c32..0000000000 --- a/Modules/Tokens/src/Tokens.Application/Tokens/Commands/AnonymizeTokenAllocationsOfIdentity/Validator.cs +++ /dev/null @@ -1,14 +0,0 @@ -using Backbone.BuildingBlocks.Application.Extensions; -using Backbone.DevelopmentKit.Identity.ValueObjects; -using FluentValidation; - -namespace Backbone.Modules.Tokens.Application.Tokens.Commands.AnonymizeTokenAllocationsOfIdentity; - -public class Validator : AbstractValidator -{ - public Validator() - { - RuleFor(q => q.IdentityAddress) - .ValidId(); - } -} diff --git a/Modules/Tokens/src/Tokens.Application/Tokens/Commands/DeleteTokenAllocationsOfIdentity/DeleteTokenAllocationsOfIdentityCommand.cs b/Modules/Tokens/src/Tokens.Application/Tokens/Commands/DeleteTokenAllocationsOfIdentity/DeleteTokenAllocationsOfIdentityCommand.cs new file mode 100644 index 0000000000..e884071fa2 --- /dev/null +++ b/Modules/Tokens/src/Tokens.Application/Tokens/Commands/DeleteTokenAllocationsOfIdentity/DeleteTokenAllocationsOfIdentityCommand.cs @@ -0,0 +1,8 @@ +using MediatR; + +namespace Backbone.Modules.Tokens.Application.Tokens.Commands.DeleteTokenAllocationsOfIdentity; + +public class DeleteTokenAllocationsOfIdentityCommand : IRequest +{ + public required string IdentityAddress { get; init; } +} diff --git a/Modules/Tokens/src/Tokens.Application/Tokens/Commands/DeleteTokenAllocationsOfIdentity/Handler.cs b/Modules/Tokens/src/Tokens.Application/Tokens/Commands/DeleteTokenAllocationsOfIdentity/Handler.cs new file mode 100644 index 0000000000..cc36a37f5b --- /dev/null +++ b/Modules/Tokens/src/Tokens.Application/Tokens/Commands/DeleteTokenAllocationsOfIdentity/Handler.cs @@ -0,0 +1,26 @@ +using Backbone.DevelopmentKit.Identity.ValueObjects; +using Backbone.Modules.Tokens.Application.Infrastructure.Persistence.Repository; +using Backbone.Modules.Tokens.Domain.Entities; +using MediatR; + +namespace Backbone.Modules.Tokens.Application.Tokens.Commands.DeleteTokenAllocationsOfIdentity; + +public class Handler : IRequestHandler +{ + private readonly ITokensRepository _tokensRepository; + + public Handler(ITokensRepository tokensRepository) + { + _tokensRepository = tokensRepository; + } + + public async Task Handle(DeleteTokenAllocationsOfIdentityCommand request, CancellationToken cancellationToken) + { + var tokens = (await _tokensRepository.ListWithoutContent(Token.HasAllocationFor(IdentityAddress.Parse(request.IdentityAddress)), cancellationToken, track: true)).ToList(); + + foreach (var token in tokens) + token.DeleteTokenAllocation(request.IdentityAddress); + + await _tokensRepository.Update(tokens, cancellationToken); + } +} diff --git a/Modules/Tokens/src/Tokens.Application/Tokens/Commands/DeleteTokenAllocationsOfIdentity/Validator.cs b/Modules/Tokens/src/Tokens.Application/Tokens/Commands/DeleteTokenAllocationsOfIdentity/Validator.cs new file mode 100644 index 0000000000..a46a000af5 --- /dev/null +++ b/Modules/Tokens/src/Tokens.Application/Tokens/Commands/DeleteTokenAllocationsOfIdentity/Validator.cs @@ -0,0 +1,14 @@ +using Backbone.BuildingBlocks.Application.Extensions; +using Backbone.DevelopmentKit.Identity.ValueObjects; +using FluentValidation; + +namespace Backbone.Modules.Tokens.Application.Tokens.Commands.DeleteTokenAllocationsOfIdentity; + +public class Validator : AbstractValidator +{ + public Validator() + { + RuleFor(q => q.IdentityAddress) + .ValidId(); + } +} diff --git a/Modules/Tokens/src/Tokens.Domain/Entities/Token.cs b/Modules/Tokens/src/Tokens.Domain/Entities/Token.cs index bf7c5759ad..995defb6f0 100644 --- a/Modules/Tokens/src/Tokens.Domain/Entities/Token.cs +++ b/Modules/Tokens/src/Tokens.Domain/Entities/Token.cs @@ -155,13 +155,11 @@ private void EnsureIsPersonalized() throw new DomainException(DomainErrors.TokenNotPersonalized()); } - public void AnonymizeTokenAllocation(IdentityAddress address, string didDomainName) + public void DeleteTokenAllocation(IdentityAddress address) { var tokenAllocation = _allocations.Find(a => a.AllocatedBy == address) ?? throw new DomainException(DomainErrors.NoAllocationForIdentity()); - var anonymousIdentity = IdentityAddress.GetAnonymized(didDomainName); - - tokenAllocation.AllocatedBy = anonymousIdentity; + _allocations.Remove(tokenAllocation); } public void EnsureCanBeDeletedBy(IdentityAddress identityAddress) From a7f481f2cb50973c25bdaa45535dce2f7e2d5329 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Fri, 22 Aug 2025 13:14:54 +0200 Subject: [PATCH 13/51] chore: Add multi-run config for the identity deletion verifier workflow --- .run/Identity Deletion verifier.run.xml | 55 +++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .run/Identity Deletion verifier.run.xml diff --git a/.run/Identity Deletion verifier.run.xml b/.run/Identity Deletion verifier.run.xml new file mode 100644 index 0000000000..26f3349867 --- /dev/null +++ b/.run/Identity Deletion verifier.run.xml @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + From 63d9e25b0526dabea0b9b51ea1ebd7db167d7628 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Fri, 22 Aug 2025 13:19:54 +0200 Subject: [PATCH 14/51] feat: Add datawallets --- .../Commands/InitCommand.cs | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/InitCommand.cs b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/InitCommand.cs index 2d2d68f436..74364c6449 100644 --- a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/InitCommand.cs +++ b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/InitCommand.cs @@ -1,7 +1,6 @@ using System.CommandLine; using System.Net; using System.Text; -using System.Text.Unicode; using Backbone.AdminApi.Sdk.Endpoints.Identities.Types.Requests; using Backbone.BuildingBlocks.SDK.Endpoints.Common.Types; using Backbone.ConsumerApi.Sdk; @@ -10,7 +9,6 @@ using Backbone.ConsumerApi.Sdk.Endpoints.Identities.Types.Requests; using Backbone.ConsumerApi.Sdk.Endpoints.Messages.Types.Requests; using Backbone.ConsumerApi.Sdk.Endpoints.PushNotifications.Types.Requests; -using Backbone.ConsumerApi.Sdk.Endpoints.Relationships.Types; using Backbone.ConsumerApi.Sdk.Endpoints.Relationships.Types.Requests; using Backbone.ConsumerApi.Sdk.Endpoints.RelationshipTemplates.Types.Requests; using Backbone.ConsumerApi.Sdk.Endpoints.SyncRuns.Types.Requests; @@ -18,7 +16,6 @@ using Backbone.ConsumerApi.Sdk.Endpoints.Tokens.Types.Requests; using Backbone.Crypto; using Backbone.IdentityDeletionVerifier.Extensions; -using Backbone.Tooling; using Backbone.Tooling.Extensions; using Spectre.Console; using AdminClient = Backbone.AdminApi.Sdk.Client; @@ -175,20 +172,28 @@ private async Task CreateTokens(Client a, Client b) Password = null }); - //var allocationResponse = tAllocatedByA.IsSuccess ? await a.Tokens.GetToken(tAllocatedByA.Result!.Id) : DummyErrorResponse(); + var allocationResponse = tAllocatedByA.IsSuccess ? await a.Tokens.GetToken(tAllocatedByA.Result!.Id) : DummyErrorResponse(); - return AnsiConsole.Console.WriteResult(tA, tForA, tAllocatedByA /*, allocationResponse*/); + return AnsiConsole.Console.WriteResult(tA, tForA, tAllocatedByA, allocationResponse); } private async Task CreateDatawallet(Client a) { AnsiConsole.WriteLine("Creating datawallet..."); - AnsiConsole.WriteLine("Not yet implemented"); - //TODO: Timo (How to create a datawallet?) - await Task.CompletedTask; + var createSyncRunResponse = await a.SyncRuns.StartSyncRun(new StartSyncRunRequest + { + Duration = null, + Type = SyncRunType.DatawalletVersionUpgrade + }, 1); + + var finalizeDatawalletVersionUpgradeResponse = await a.SyncRuns.FinalizeDatawalletVersionUpgrade(createSyncRunResponse.Result!.SyncRun.Id, new FinalizeDatawalletVersionUpgradeRequest + { + NewDatawalletVersion = 1, + DatawalletModifications = [] + }); - return true; + return AnsiConsole.Console.WriteResult(createSyncRunResponse, finalizeDatawalletVersionUpgradeResponse); } private async Task StartSyncRun(Client a) @@ -247,7 +252,7 @@ private async Task StartDeletionProcesses(Client a, Client b) var responseA = await a.Identities.StartDeletionProcess(new StartDeletionProcessRequest { LengthOfGracePeriodInDays = 0 }); var responseB = await b.Identities.StartDeletionProcess(new StartDeletionProcessRequest { LengthOfGracePeriodInDays = 0.1 }); - //TODO Timo: Until the Identity Deletion Bug PR is merged, A and B can't be deleted at the same time (therefore the short grace period) + //We need a short grace period for B, so that the templates and tokens for A get anonymized instead of deleted return AnsiConsole.Console.WriteResult(responseA, responseB); } From e329e5f00a4f90b3983ecb3d8eded1f9fbbf161a Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 9 Sep 2025 13:52:50 +0200 Subject: [PATCH 15/51] chore: Remove old IdentityDeletionVerifier project --- .run/Identity Deletion verifier.run.xml | 55 ---- .../Commands/CheckCommand.cs | 117 ------- .../Commands/InitCommand.cs | 295 ------------------ .../Extensions/AnsiConsoleExtensions.cs | 16 - .../src/IdentityDeletionVerifier/FilePaths.cs | 14 - .../IdentityDeletionVerifier.csproj | 20 -- .../src/IdentityDeletionVerifier/Program.cs | 15 - .../Properties/launchSettings.json | 13 - Backbone.sln | 13 - 9 files changed, 558 deletions(-) delete mode 100644 .run/Identity Deletion verifier.run.xml delete mode 100644 Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/CheckCommand.cs delete mode 100644 Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/InitCommand.cs delete mode 100644 Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Extensions/AnsiConsoleExtensions.cs delete mode 100644 Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/FilePaths.cs delete mode 100644 Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj delete mode 100644 Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Program.cs delete mode 100644 Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Properties/launchSettings.json diff --git a/.run/Identity Deletion verifier.run.xml b/.run/Identity Deletion verifier.run.xml deleted file mode 100644 index 26f3349867..0000000000 --- a/.run/Identity Deletion verifier.run.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/CheckCommand.cs b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/CheckCommand.cs deleted file mode 100644 index 56328b5982..0000000000 --- a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/CheckCommand.cs +++ /dev/null @@ -1,117 +0,0 @@ -using System.CommandLine; -using System.IO.Compression; -using System.Text; -using Spectre.Console; - -namespace Backbone.IdentityDeletionVerifier.Commands; - -public class CheckCommand : Command -{ - public CheckCommand() : base("check", "Check the exported database file for the given identity address in the temp directory") - { - SetAction(Handle); - } - - private async Task Handle(ParseResult _, CancellationToken cancellationToken) - { - if (!DirectoryExists()) - { - AnsiConsole.MarkupLineInterpolated($"[red bold]The temp directory[/][grey bold]{FilePaths.PATH_TO_TEMP_DIR} [/][red bold]doesn't exist.[/]"); - return -1; - } - - if (!IdentitiesFileExists()) - { - AnsiConsole.MarkupLine("[red bold]The deleted identities file doesn't exist. Run the Init command first.[/]"); - return -1; - } - - if (!ExportFileExists()) - { - AnsiConsole.MarkupLine("[red bold]No exported database file found. Run the Admin Cli Export Database command first.[/]"); - return -1; - } - - var a = await GetIdentityToCheck(); - if (a == null) - { - AnsiConsole.MarkupLineInterpolated($"[red bold]The identities file couldn't be read or has no Identity[/]"); - return -1; - } - - AnsiConsole.MarkupLineInterpolated($"[green bold]Identity to check:[/] [grey bold]{a}[/]"); - - return await CheckExportFileForIdentities(GetLatestExportFile(), a); - } - - private bool DirectoryExists() => Directory.Exists(FilePaths.PATH_TO_TEMP_DIR); - private bool IdentitiesFileExists() => File.Exists(FilePaths.PATH_TO_IDENTITIES_FILE); - private bool ExportFileExists() => Directory.EnumerateFiles(FilePaths.PATH_TO_TEMP_DIR).Any(FilePaths.EXPORT_FILE_PATTERN.IsMatch); - private string GetLatestExportFile() => Directory.EnumerateFiles(FilePaths.PATH_TO_TEMP_DIR).Where(e => FilePaths.EXPORT_FILE_PATTERN.IsMatch(e)).Max()!; - - private async Task GetIdentityToCheck() - { - using var reader = new StreamReader(File.OpenRead(FilePaths.PATH_TO_IDENTITIES_FILE), Encoding.UTF8); - - return await reader.ReadLineAsync(); - } - - private async Task CheckExportFileForIdentities(string exportFile, string identityToCheck) - { - return await AnsiConsole.Progress() - .AutoClear(false) - .HideCompleted(false) - .Columns( - new TaskDescriptionColumn(), - new ProgressBarColumn(), - new PercentageColumn(), - new RemainingTimeColumn(), - new SpinnerColumn(Spinner.Known.Clock) - ) - .StartAsync(async ctx => - { - using var archive = ZipFile.OpenRead(exportFile); - - var found = 0; - var tasks = archive.Entries - .Select(e => ctx.AddTask(e.Name, autoStart: false)) - .ToList(); - - foreach (var (index, entry) in archive.Entries.Index()) - { - var task = tasks[index]; - task.StartTask(); - found += await CheckCsvFileForIdentity(entry, identityToCheck, task); - } - - return found; - }); - } - - private async Task CheckCsvFileForIdentity(ZipArchiveEntry file, string identityToCheck, ProgressTask progressReporter) - { - using var reader = new StreamReader(file.Open(), Encoding.UTF8); - List lines = []; - var found = 0; - - while (!reader.EndOfStream) - lines.Add(await reader.ReadLineAsync() ?? string.Empty); - - progressReporter.MaxValue = lines.Count; - - foreach (var line in lines) - { - var count = line - .Split(',') - .Count(s => string.Equals(s, identityToCheck, StringComparison.OrdinalIgnoreCase)); - - if (count != 0) - AnsiConsole.MarkupLineInterpolated($"[red bold]Found[/] [grey bold]{count}[/] [red bold]occurrences in[/] [grey bold]{file.Name}[/][red bold]:[/] [grey bold]{line}[/]"); - - found += count; - progressReporter.Increment(1); - } - - return found; - } -} diff --git a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/InitCommand.cs b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/InitCommand.cs deleted file mode 100644 index 74364c6449..0000000000 --- a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/InitCommand.cs +++ /dev/null @@ -1,295 +0,0 @@ -using System.CommandLine; -using System.Net; -using System.Text; -using Backbone.AdminApi.Sdk.Endpoints.Identities.Types.Requests; -using Backbone.BuildingBlocks.SDK.Endpoints.Common.Types; -using Backbone.ConsumerApi.Sdk; -using Backbone.ConsumerApi.Sdk.Authentication; -using Backbone.ConsumerApi.Sdk.Endpoints.Files.Types.Requests; -using Backbone.ConsumerApi.Sdk.Endpoints.Identities.Types.Requests; -using Backbone.ConsumerApi.Sdk.Endpoints.Messages.Types.Requests; -using Backbone.ConsumerApi.Sdk.Endpoints.PushNotifications.Types.Requests; -using Backbone.ConsumerApi.Sdk.Endpoints.Relationships.Types.Requests; -using Backbone.ConsumerApi.Sdk.Endpoints.RelationshipTemplates.Types.Requests; -using Backbone.ConsumerApi.Sdk.Endpoints.SyncRuns.Types.Requests; -using Backbone.ConsumerApi.Sdk.Endpoints.Tokens.Types; -using Backbone.ConsumerApi.Sdk.Endpoints.Tokens.Types.Requests; -using Backbone.Crypto; -using Backbone.IdentityDeletionVerifier.Extensions; -using Backbone.Tooling.Extensions; -using Spectre.Console; -using AdminClient = Backbone.AdminApi.Sdk.Client; - -namespace Backbone.IdentityDeletionVerifier.Commands; - -public class InitCommand : Command -{ - private static readonly ClientCredentials CREDENTIALS = new("test", "test"); - private const string PASSWORD = "password"; - private const string ADMIN_API_KEY = "test"; - private const string DUMMY_STRING = "AAAA"; - private static readonly byte[] DUMMY_DATA = DUMMY_STRING.GetBytes(); - - public InitCommand() : base("init", "Creates an identity with relationships, messages etc. and starts an immediate deletion process") - { - var consumerBaseUrl = new Option("--consumerBaseUrl") - { - Required = true, - Description = "The base url for the Consumer Api" - }; - - var adminBaseUrl = new Option("--adminBaseUrl") - { - Required = true, - Description = "The base url for the Admin Api" - }; - - Options.Add(consumerBaseUrl); - Options.Add(adminBaseUrl); - SetAction((result, _) => Handle(result.GetRequiredValue(consumerBaseUrl), result.GetRequiredValue(adminBaseUrl))); - } - - private async Task Handle(string consumerBaseUrl, string adminBaseUrl) - { - List results = []; - - var ex = await AnsiConsole.Status() - .Spinner(Spinner.Known.Clock) - .StartAsync("Preparing the identity deletion", async _ => - { - try - { - var (a, b, admin) = await CreateClients(consumerBaseUrl, adminBaseUrl); - results.Add(await CreateChallenge(a)); - results.Add(await CreateFile(a)); - results.Add(await CreatePushNotificationHandle(a)); - results.Add(await CreateIndividualQuota(a, admin)); - results.Add(await CreateTokens(a, b)); - results.Add(await CreateDatawallet(a)); - results.Add(await StartSyncRun(a)); - results.Add(await CreateRelationship(a, b)); - results.Add(await SendMessage(a, b)); - - results.Add(await StartDeletionProcesses(a, b)); - - await WriteIdentitiesToFile(a, b); - } - catch (Exception e) - { - AnsiConsole.WriteException(e); - return false; - } - - return true; - }); - results.Add(ex); - - return results.Contains(false) ? 1 : 0; - } - - private async Task<(Client, Client, AdminClient)> CreateClients(string consumerBaseUrl, string adminBaseUrl) - { - AnsiConsole.WriteLine("Creating clients..."); - var a = await Client.CreateForNewIdentity(consumerBaseUrl, CREDENTIALS, PASSWORD); - var b = await Client.CreateForNewIdentity(consumerBaseUrl, CREDENTIALS, PASSWORD); - var admin = AdminClient.Create(adminBaseUrl, ADMIN_API_KEY); - - AnsiConsole.MarkupLineInterpolated($"A: [green bold]{a.IdentityData?.Address}[/]"); - AnsiConsole.MarkupLineInterpolated($"B: [green bold]{b.IdentityData?.Address}[/]"); - AnsiConsole.WriteLine(Emoji.Known.CheckMarkButton); - - return (a, b, admin); - } - - private async Task CreateChallenge(Client a) - { - AnsiConsole.WriteLine("Creating challenge..."); - var response = await a.Challenges.CreateChallenge(); - return AnsiConsole.Console.WriteResult(response); - } - - private async Task CreateFile(Client a) - { - AnsiConsole.WriteLine("Creating file..."); - var response = await a.Files.UploadFile(new CreateFileRequest - { - Content = new MemoryStream("Content".GetBytes()), - Owner = a.IdentityData!.Address, - OwnerSignature = DUMMY_STRING, - CipherHash = DUMMY_STRING, - ExpiresAt = DateTime.UtcNow.AddDays(1), - EncryptedProperties = DUMMY_STRING - }); - return AnsiConsole.Console.WriteResult(response); - } - - private async Task CreatePushNotificationHandle(Client a) - { - AnsiConsole.WriteLine("Creating push notification handle..."); - var response = await a.PushNotifications.RegisterForPushNotifications(new UpdateDeviceRegistrationRequest - { - AppId = "de.bildungsraum.wallet.experimental", - Handle = "asdsdaasdasdasds", - Platform = "dummy" - }); - return AnsiConsole.Console.WriteResult(response); - } - - private async Task CreateIndividualQuota(Client a, AdminClient admin) - { - AnsiConsole.WriteLine("Creating individual quota..."); - var response = await admin.Identities.CreateIndividualQuota(a.IdentityData!.Address, new CreateQuotaForIdentityRequest - { - Max = 100, - MetricKey = "NumberOfSentMessages", - Period = "hour" - }); - return AnsiConsole.Console.WriteResult(response); - } - - private async Task CreateTokens(Client a, Client b) - { - AnsiConsole.WriteLine("Creating tokens (one from A, one for A, one allocated by A)..."); - var tA = await a.Tokens.CreateToken(new CreateTokenRequest - { - Content = DUMMY_DATA, - ExpiresAt = DateTime.UtcNow.AddDays(1), - ForIdentity = null, - Password = null - }); - var tForA = await b.Tokens.CreateToken(new CreateTokenRequest - { - Content = DUMMY_DATA, - ExpiresAt = DateTime.UtcNow.AddDays(1), - ForIdentity = a.IdentityData!.Address, - Password = null - }); - var tAllocatedByA = await b.Tokens.CreateToken(new CreateTokenRequest - { - Content = DUMMY_DATA, - ExpiresAt = DateTime.UtcNow.AddDays(1), - ForIdentity = a.IdentityData!.Address, - Password = null - }); - - var allocationResponse = tAllocatedByA.IsSuccess ? await a.Tokens.GetToken(tAllocatedByA.Result!.Id) : DummyErrorResponse(); - - return AnsiConsole.Console.WriteResult(tA, tForA, tAllocatedByA, allocationResponse); - } - - private async Task CreateDatawallet(Client a) - { - AnsiConsole.WriteLine("Creating datawallet..."); - - var createSyncRunResponse = await a.SyncRuns.StartSyncRun(new StartSyncRunRequest - { - Duration = null, - Type = SyncRunType.DatawalletVersionUpgrade - }, 1); - - var finalizeDatawalletVersionUpgradeResponse = await a.SyncRuns.FinalizeDatawalletVersionUpgrade(createSyncRunResponse.Result!.SyncRun.Id, new FinalizeDatawalletVersionUpgradeRequest - { - NewDatawalletVersion = 1, - DatawalletModifications = [] - }); - - return AnsiConsole.Console.WriteResult(createSyncRunResponse, finalizeDatawalletVersionUpgradeResponse); - } - - private async Task StartSyncRun(Client a) - { - AnsiConsole.WriteLine("Starting sync run..."); - var response = await a.SyncRuns.StartSyncRun(new StartSyncRunRequest - { - Duration = null, - Type = SyncRunType.ExternalEventSync - }, 1); - return AnsiConsole.Console.WriteResult(response); - } - - private async Task CreateRelationship(Client a, Client b) - { - AnsiConsole.WriteLine("Creating relationship..."); - var templateForA = await b.RelationshipTemplates.CreateTemplate(new CreateRelationshipTemplateRequest { Content = DUMMY_DATA, ForIdentity = a.IdentityData!.Address }); - if (templateForA.IsError) return false; - - var templateByA = await a.RelationshipTemplates.CreateTemplate(new CreateRelationshipTemplateRequest { Content = DUMMY_DATA }); - - var templateAllocatedByA = await b.RelationshipTemplates.CreateTemplate(new CreateRelationshipTemplateRequest { Content = DUMMY_DATA }); - if (templateAllocatedByA.IsError) return false; - var getTemplateResponse = await a.RelationshipTemplates.GetTemplate(templateAllocatedByA.Result!.Id); - - var createRelationshipResponse = await a.Relationships.CreateRelationship(new CreateRelationshipRequest { Content = DUMMY_DATA, RelationshipTemplateId = templateForA.Result!.Id }); - if (createRelationshipResponse.IsError) return false; - - var acceptRelationshipResponse = await b.Relationships.AcceptRelationship(createRelationshipResponse.Result!.Id, new AcceptRelationshipRequest { CreationResponseContent = DUMMY_DATA }); - - return AnsiConsole.Console.WriteResult(templateForA, templateByA, templateAllocatedByA, getTemplateResponse, createRelationshipResponse, acceptRelationshipResponse); - } - - private async Task SendMessage(Client a, Client b) - { - AnsiConsole.WriteLine("Sending message from A to B..."); - var sendMessageResponse = await a.Messages.SendMessage(new SendMessageRequest - { - Attachments = [], - Body = "Message".GetBytes(), - Recipients = - [ - new SendMessageRequestRecipientInformation - { - Address = b.IdentityData!.Address, - EncryptedKey = ConvertibleString.FromUtf8("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA").BytesRepresentation - } - ] - }); - return AnsiConsole.Console.WriteResult(sendMessageResponse); - } - - private async Task StartDeletionProcesses(Client a, Client b) - { - AnsiConsole.WriteLine("Starting deletion processes..."); - - var responseA = await a.Identities.StartDeletionProcess(new StartDeletionProcessRequest { LengthOfGracePeriodInDays = 0 }); - var responseB = await b.Identities.StartDeletionProcess(new StartDeletionProcessRequest { LengthOfGracePeriodInDays = 0.1 }); - //We need a short grace period for B, so that the templates and tokens for A get anonymized instead of deleted - - return AnsiConsole.Console.WriteResult(responseA, responseB); - } - - private async Task WriteIdentitiesToFile(Client a, Client b) - { - AnsiConsole.WriteLine("Writing identity addresses to a temp file..."); - - if (!Directory.Exists(FilePaths.PATH_TO_TEMP_DIR)) - Directory.CreateDirectory(FilePaths.PATH_TO_TEMP_DIR); - - var file = new StreamWriter(new FileStream(FilePaths.PATH_TO_IDENTITIES_FILE, FileMode.Create), Encoding.UTF8); - - await file.WriteLineAsync(a.IdentityData!.Address); - await file.WriteLineAsync(b.IdentityData!.Address); - - await file.FlushAsync(); - file.Close(); - - AnsiConsole.WriteLine(Emoji.Known.CheckMarkButton); - } - - private static ApiResponse DummyErrorResponse() - { - return new ApiResponse - { - ContentType = null, - Error = new ApiError - { - Code = "error.dummy", - Data = null, - Message = "An error occured", - Docs = "", - Id = "", - Time = DateTime.UtcNow - }, - Status = HttpStatusCode.BadRequest - }; - } -} diff --git a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Extensions/AnsiConsoleExtensions.cs b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Extensions/AnsiConsoleExtensions.cs deleted file mode 100644 index 5f3abeb21f..0000000000 --- a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Extensions/AnsiConsoleExtensions.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Backbone.BuildingBlocks.SDK.Endpoints.Common.Types; -using Backbone.Tooling.Extensions; -using Spectre.Console; - -namespace Backbone.IdentityDeletionVerifier.Extensions; - -public static class AnsiConsoleExtensions -{ - public static bool WriteResult(this IAnsiConsole console, params IResponse[] responses) - { - var success = responses.All(r => r.IsSuccess); - console.WriteLine(success ? Emoji.Known.CheckMarkButton : Emoji.Known.CrossMark); - - return success; - } -} diff --git a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/FilePaths.cs b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/FilePaths.cs deleted file mode 100644 index 25cd01e83c..0000000000 --- a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/FilePaths.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System.Text.RegularExpressions; - -namespace Backbone.IdentityDeletionVerifier; - -public static partial class FilePaths -{ - public static readonly string PATH_TO_TEMP_DIR = Path.Combine(Path.GetTempPath(), "enmeshed", "backbone"); - public const string IDENTITIES_FILENAME = "deleted-identities.txt"; - public static readonly Regex EXPORT_FILE_PATTERN = MyRegex(); - public static readonly string PATH_TO_IDENTITIES_FILE = Path.Combine(PATH_TO_TEMP_DIR, IDENTITIES_FILENAME); - - [GeneratedRegex(@"export-\d{8}_\d{6}\.zip$")] - private static partial Regex MyRegex(); -} diff --git a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj deleted file mode 100644 index 85da230e4c..0000000000 --- a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj +++ /dev/null @@ -1,20 +0,0 @@ - - - - Exe - net9.0 - enable - enable - - - - - - - - - - - - - diff --git a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Program.cs b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Program.cs deleted file mode 100644 index 495c0b142c..0000000000 --- a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Program.cs +++ /dev/null @@ -1,15 +0,0 @@ -// See https://aka.ms/new-console-template for more information - -using System.CommandLine; -using System.Text; -using Backbone.IdentityDeletionVerifier.Commands; - -Console.OutputEncoding = Encoding.UTF8; - -var command = new RootCommand -{ - new InitCommand(), - new CheckCommand() -}; - -return await command.Parse(args).InvokeAsync(); diff --git a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Properties/launchSettings.json b/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Properties/launchSettings.json deleted file mode 100644 index d0016ab078..0000000000 --- a/Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Properties/launchSettings.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "$schema": "http://json.schemastore.org/launchsettings.json", - "profiles": { - "Init": { - "commandName": "Project", - "commandLineArgs": "init --consumerBaseUrl http://localhost:8081 --adminBaseUrl http://localhost:8082" - }, - "Check": { - "commandName": "Project", - "commandLineArgs": "check" - } - } -} diff --git a/Backbone.sln b/Backbone.sln index 7a09ad5e71..3b6b12209d 100644 --- a/Backbone.sln +++ b/Backbone.sln @@ -396,12 +396,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BuildingBlocks.Module", "Bu EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Announcements.Domain.Tests", "Modules\Announcements\test\Announcements.Domain.Tests\Announcements.Domain.Tests.csproj", "{44300266-A7BD-4D04-A719-6EAAEFEB1F95}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "IdentityDeletionVerifier", "IdentityDeletionVerifier", "{584BC088-D517-4799-80DF-73078E7BAC2F}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{E8D1B480-AE46-4DE0-BD74-EDC70E95B44C}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "IdentityDeletionVerifier", "Applications\IdentityDeletionVerifier\src\IdentityDeletionVerifier\IdentityDeletionVerifier.csproj", "{DC3E495C-60E8-4885-86D6-73466893CBB4}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -884,10 +878,6 @@ Global {44300266-A7BD-4D04-A719-6EAAEFEB1F95}.Debug|Any CPU.Build.0 = Debug|Any CPU {44300266-A7BD-4D04-A719-6EAAEFEB1F95}.Release|Any CPU.ActiveCfg = Release|Any CPU {44300266-A7BD-4D04-A719-6EAAEFEB1F95}.Release|Any CPU.Build.0 = Release|Any CPU - {DC3E495C-60E8-4885-86D6-73466893CBB4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DC3E495C-60E8-4885-86D6-73466893CBB4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DC3E495C-60E8-4885-86D6-73466893CBB4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {DC3E495C-60E8-4885-86D6-73466893CBB4}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1071,9 +1061,6 @@ Global {74CDB906-8BC9-42F7-A3FA-2B675A240D51} = {399BA0C2-C130-4C8E-8F2D-8BB45AB9FD1A} {0462B097-733E-4CC0-945D-664AECCA04C3} = {06D714AE-EDF4-421C-9340-EDA6FCDF491F} {44300266-A7BD-4D04-A719-6EAAEFEB1F95} = {399BA0C2-C130-4C8E-8F2D-8BB45AB9FD1A} - {584BC088-D517-4799-80DF-73078E7BAC2F} = {44C9D62D-813D-497A-8DDF-C06E515CB22E} - {E8D1B480-AE46-4DE0-BD74-EDC70E95B44C} = {584BC088-D517-4799-80DF-73078E7BAC2F} - {DC3E495C-60E8-4885-86D6-73466893CBB4} = {E8D1B480-AE46-4DE0-BD74-EDC70E95B44C} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {1F3BD2C6-7CB3-450F-A21A-23EA520D5B7A} From fc34241dd70fb4cef11fb2a3c28c9455413edf7e Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 9 Sep 2025 13:54:40 +0200 Subject: [PATCH 16/51] fix: Fix syntax and output file of SqlServer dump script --- scripts/windows/dumps/dump-sqlserver.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/windows/dumps/dump-sqlserver.ps1 b/scripts/windows/dumps/dump-sqlserver.ps1 index 12cfc61a61..b700b61e83 100644 --- a/scripts/windows/dumps/dump-sqlserver.ps1 +++ b/scripts/windows/dumps/dump-sqlserver.ps1 @@ -2,13 +2,13 @@ param ( [string]$Hostname = "host.docker.internal", [string]$Username = "SA", - [string]$Password = "Passw0rd" - [string]$DbName = "enmeshed", + [string]$Password = "Passw0rd", + [string]$DbName = "enmeshed" ) $DumpFile = "enmeshed.bacpac" -docker run --rm -v "$PSScriptRoot\dump-files:/dump" ormico/sqlpackage sqlpackage /Action:Export /SourceServerName:$Hostname /SourceDatabaseName:$DbName /TargetFile:/tmp/$DumpFile /SourceUser:$Username /SourcePassword:$Password /SourceTrustServerCertificate:True +docker run --rm -v "$PSScriptRoot\dump-files:/dump" ormico/sqlpackage sqlpackage /Action:Export /SourceServerName:$Hostname /SourceDatabaseName:$DbName /TargetFile:/dump/$DumpFile /SourceUser:$Username /SourcePassword:$Password /SourceTrustServerCertificate:True if ($LASTEXITCODE -ne 0) { throw "Error: Database export to $DumpFile failed." From 8ba8f494acaf1555d3dfe4e1ab1c4700d6dddb48 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 9 Sep 2025 13:57:39 +0200 Subject: [PATCH 17/51] chore: Import PgDump and SMO --- .../src/Job.IdentityDeletion/Job.IdentityDeletion.csproj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Job.IdentityDeletion.csproj b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Job.IdentityDeletion.csproj index d00c61c7c2..1de081280a 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Job.IdentityDeletion.csproj +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Job.IdentityDeletion.csproj @@ -14,6 +14,8 @@ + + From 79e98d698d51449723ac92a37698f6dcb1886023 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 9 Sep 2025 13:58:11 +0200 Subject: [PATCH 18/51] feat: Add database exporters --- .../Exporters/IDbExporter.cs | 6 ++ .../Exporters/PostgresDbExporter.cs | 37 +++++++++++ .../Exporters/SqlServerDbExporter.cs | 66 +++++++++++++++++++ .../Tests/DummyClasses/DummyDbExporter.cs | 8 +++ 4 files changed, 117 insertions(+) create mode 100644 Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Exporters/IDbExporter.cs create mode 100644 Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Exporters/PostgresDbExporter.cs create mode 100644 Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Exporters/SqlServerDbExporter.cs create mode 100644 Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/DummyClasses/DummyDbExporter.cs diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Exporters/IDbExporter.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Exporters/IDbExporter.cs new file mode 100644 index 0000000000..c47c89f509 --- /dev/null +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Exporters/IDbExporter.cs @@ -0,0 +1,6 @@ +namespace Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Exporters; + +public interface IDbExporter +{ + Task ExportDb(string targetFile, CancellationToken cancellationToken); +} diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Exporters/PostgresDbExporter.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Exporters/PostgresDbExporter.cs new file mode 100644 index 0000000000..e2838befb4 --- /dev/null +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Exporters/PostgresDbExporter.cs @@ -0,0 +1,37 @@ +using Backbone.BuildingBlocks.Infrastructure.Persistence.Database; +using Backbone.Tooling.Extensions; +using Microsoft.Extensions.Options; +using PgDump; + +namespace Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Exporters; + +public class PostgresDbExporter : IDbExporter +{ + private readonly DatabaseConfiguration _configuration; + private readonly ILogger _logger; + + public PostgresDbExporter(IOptions configuration, ILogger logger) + { + _configuration = configuration.Value.Infrastructure.SqlDatabase; + _logger = logger; + } + + public async Task ExportDb(string targetFile, CancellationToken cancellationToken) + { + _logger.LogInformation("Starting Postgres database dump"); + + var parts = _configuration.ConnectionString.Split(';'); + var host = parts.ExtractString("Server="); + var port = int.Parse(parts.ExtractString("Port=")); + var username = parts.ExtractString("User ID="); + var password = parts.ExtractString("Password="); + var database = parts.ExtractString("Database="); + + var connection = new ConnectionOptions(host, port, username, password, database); + var client = new PgClient(connection); + + await client.DumpAsync(new FileOutputProvider(targetFile), 1.Minutes(), DumpFormat.Plain, cancellationToken); + + _logger.LogInformation("Postgres database dump completed"); + } +} diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Exporters/SqlServerDbExporter.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Exporters/SqlServerDbExporter.cs new file mode 100644 index 0000000000..577da8e051 --- /dev/null +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Exporters/SqlServerDbExporter.cs @@ -0,0 +1,66 @@ +using System.Text; +using Backbone.BuildingBlocks.Infrastructure.Persistence.Database; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Options; +using Microsoft.SqlServer.Management.Common; +using Microsoft.SqlServer.Management.Smo; + +namespace Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Exporters; + +public class SqlServerDbExporter : IDbExporter +{ + private readonly DatabaseConfiguration _configuration; + private readonly ILogger _logger; + + public SqlServerDbExporter(IOptions configuration, ILogger logger) + { + _configuration = configuration.Value.Infrastructure.SqlDatabase; + _logger = logger; + } + + public async Task ExportDb(string targetFile, CancellationToken cancellationToken) + { + _logger.LogInformation("Starting Sql Server database dump"); + + var parts = _configuration.ConnectionString.Split(';'); + var databaseName = parts.ExtractString("Database="); + var server = new Server(new ServerConnection(new SqlConnection(_configuration.ConnectionString))); + var database = server.Databases[databaseName] ?? throw new ArgumentException($"No database with name \"{databaseName}\" found in sql server"); + var scripter = new Scripter(server) + { + Options = new ScriptingOptions + { + EnforceScriptingOptions = true, + WithDependencies = false, + IncludeHeaders = false, + ScriptDrops = false, + AppendToFile = false, + ScriptSchema = false, + ScriptData = true, + IncludeIfNotExists = false, + Default = true, + Indexes = false + } + }; + + await using var writer = new StreamWriter(new FileStream(targetFile, FileMode.Create), Encoding.UTF8); + foreach (var table in database.Tables.Cast()) + { + if (table.IsSystemObject) continue; + + _logger.LogInformation("Writing table {schema}.{table}", table.Schema, table.Name); + + await writer.WriteLineAsync($"table;{table.Schema};{table.Name}"); + + foreach (var line in scripter.EnumScript(new[] { table.Urn })) + { + await writer.WriteLineAsync(line); + } + + await writer.WriteLineAsync(); + await writer.FlushAsync(cancellationToken); + } + + _logger.LogInformation("Sql Server database dump completed"); + } +} diff --git a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/DummyClasses/DummyDbExporter.cs b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/DummyClasses/DummyDbExporter.cs new file mode 100644 index 0000000000..6b85c72632 --- /dev/null +++ b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/DummyClasses/DummyDbExporter.cs @@ -0,0 +1,8 @@ +using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Exporters; + +namespace Backbone.Job.IdentityDeletion.Tests.Tests.DummyClasses; + +public class DummyDbExporter : IDbExporter +{ + public Task ExportDb(string targetFile, CancellationToken cancellationToken) => Task.CompletedTask; +} From 5c73924783b228ef4f373c6d25bb86333635e48f Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 9 Sep 2025 13:58:32 +0200 Subject: [PATCH 19/51] feat: Add Sql extractors --- .../Extractors/ISqlExtractor.cs | 18 +++++ .../Extractors/PostgresSqlExtractor.cs | 70 +++++++++++++++++++ .../Extractors/SqlserverSqlExtractor.cs | 61 ++++++++++++++++ .../Tests/DummyClasses/DummySqlExtractor.cs | 21 ++++++ 4 files changed, 170 insertions(+) create mode 100644 Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Extractors/ISqlExtractor.cs create mode 100644 Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Extractors/PostgresSqlExtractor.cs create mode 100644 Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Extractors/SqlserverSqlExtractor.cs create mode 100644 Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/DummyClasses/DummySqlExtractor.cs diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Extractors/ISqlExtractor.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Extractors/ISqlExtractor.cs new file mode 100644 index 0000000000..573fb14f0c --- /dev/null +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Extractors/ISqlExtractor.cs @@ -0,0 +1,18 @@ +namespace Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Extractors; + +public interface ISqlExtractor +{ + IAsyncEnumerable ExtractTables(string file); +} + +public record ExtractedTable +{ + public required TableId Id { get; init; } + public required IAsyncEnumerable EntryLines { get; init; } +} + +public record TableId +{ + public required string Schema { get; init; } + public required string Table { get; init; } +} diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Extractors/PostgresSqlExtractor.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Extractors/PostgresSqlExtractor.cs new file mode 100644 index 0000000000..918bca5c4b --- /dev/null +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Extractors/PostgresSqlExtractor.cs @@ -0,0 +1,70 @@ +using System.Text; +using System.Text.RegularExpressions; +using Backbone.Tooling.Extensions; + +namespace Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Extractors; + +public partial class PostgresSqlExtractor : ISqlExtractor +{ + [GeneratedRegex(""" + \"[^\"]*\" + """)] + private static partial Regex TableNameAndSchemaRegex(); + + private readonly ILogger _logger; + + public PostgresSqlExtractor(ILogger logger) + { + _logger = logger; + } + + public async IAsyncEnumerable ExtractTables(string file) + { + _logger.LogInformation("Extracting dumped postgres data"); + + var reader = new StreamReader(new FileStream(file, FileMode.Open, FileAccess.Read), Encoding.UTF8); + + while (!reader.EndOfStream) + { + var line = await reader.ReadLineAsync(); + if (line.IsNullOrEmpty()) continue; + + if (line.Trim().StartsWith("COPY")) + { + var id = ExtractTableNameAndSchema(line); + var entryLines = ReadCopyLines(reader); + yield return new ExtractedTable + { + Id = id, + EntryLines = entryLines + }; + } + } + + _logger.LogInformation("Extraction complete"); + } + + private static TableId ExtractTableNameAndSchema(string startLine) + { + var matches = TableNameAndSchemaRegex().Matches(startLine); + var schemaName = matches[0].Value.Trim('"'); + var tableName = matches[1].Value.Trim('"'); + + return new TableId + { + Schema = schemaName, + Table = tableName + }; + } + + private static async IAsyncEnumerable ReadCopyLines(StreamReader reader) + { + while (!reader.EndOfStream) + { + var line = await reader.ReadLineAsync(); + if (line is null or "\\.") break; + + yield return line; + } + } +} diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Extractors/SqlserverSqlExtractor.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Extractors/SqlserverSqlExtractor.cs new file mode 100644 index 0000000000..dbc1dccf29 --- /dev/null +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/Extractors/SqlserverSqlExtractor.cs @@ -0,0 +1,61 @@ +using System.Text; +using Backbone.Tooling.Extensions; + +namespace Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Extractors; + +public class SqlserverSqlExtractor : ISqlExtractor +{ + private readonly ILogger _logger; + + public SqlserverSqlExtractor(ILogger logger) + { + _logger = logger; + } + + public async IAsyncEnumerable ExtractTables(string file) + { + _logger.LogInformation("Extracting dumped sql server data"); + + var reader = new StreamReader(new FileStream(file, FileMode.Open), Encoding.UTF8); + + while (!reader.EndOfStream) + { + var line = await reader.ReadLineAsync(); + if (line.IsNullOrEmpty()) continue; + + if (line.StartsWith("table")) + { + var id = ExtractTableNameAndSchema(line); + var lines = ExtractLines(reader); + yield return new ExtractedTable + { + Id = id, + EntryLines = lines + }; + } + } + + _logger.LogInformation("Extraction complete"); + } + + private static TableId ExtractTableNameAndSchema(string line) + { + var parts = line.Split(';'); + return new TableId + { + Schema = parts[1], + Table = parts[2] + }; + } + + private static async IAsyncEnumerable ExtractLines(StreamReader reader) + { + while (!reader.EndOfStream) + { + var line = await reader.ReadLineAsync(); + if (line.IsNullOrEmpty()) break; + + yield return line; + } + } +} diff --git a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/DummyClasses/DummySqlExtractor.cs b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/DummyClasses/DummySqlExtractor.cs new file mode 100644 index 0000000000..9032acdc19 --- /dev/null +++ b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/DummyClasses/DummySqlExtractor.cs @@ -0,0 +1,21 @@ +using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Extractors; + +namespace Backbone.Job.IdentityDeletion.Tests.Tests.DummyClasses; + +public class DummySqlExtractor : ISqlExtractor +{ + public static readonly TableId TEST_ID = new() { Schema = "Test", Table = "Test" }; + + private readonly ExtractedTable _table; + + public DummySqlExtractor(List lines) + { + _table = new ExtractedTable + { + Id = TEST_ID, + EntryLines = lines.ToAsyncEnumerable() + }; + } + + public IAsyncEnumerable ExtractTables(string file) => new[] { _table }.ToAsyncEnumerable(); +} From bceecc30db0edb354bb8308b7262acd9bf50de32 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 9 Sep 2025 14:03:43 +0200 Subject: [PATCH 20/51] feat: Add deletion verifiers --- .../DeletionVerifier.cs | 105 ++++++++++++++++++ .../IDeletionVerifier.cs | 20 ++++ .../DummyClasses/DummyDeletionVerifier.cs | 29 +++++ 3 files changed, 154 insertions(+) create mode 100644 Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/DeletionVerifier.cs create mode 100644 Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/IDeletionVerifier.cs create mode 100644 Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/DummyClasses/DummyDeletionVerifier.cs diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/DeletionVerifier.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/DeletionVerifier.cs new file mode 100644 index 0000000000..2345570210 --- /dev/null +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/DeletionVerifier.cs @@ -0,0 +1,105 @@ +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; +using Backbone.DevelopmentKit.Identity.ValueObjects; +using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Exporters; +using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Extractors; + +namespace Backbone.Job.IdentityDeletion.IdentityDeletionVerifier; + +public class DeletionVerifier : IDeletionVerifier +{ + private static readonly JsonSerializerOptions SERIALIZER_OPTIONS = new() + { + IndentSize = 4, + PropertyNameCaseInsensitive = true, + WriteIndented = true + }; + + private readonly IDbExporter _dbExporter; + private readonly ISqlExtractor _sqlExtractor; + + public DeletionVerifier(IDbExporter dbExporter, ISqlExtractor sqlExtractor) + { + _dbExporter = dbExporter; + _sqlExtractor = sqlExtractor; + } + + public async Task VerifyDeletion(List addressesToVerify, CancellationToken cancellationToken) + { + await ExportDatabase(cancellationToken); + + return await CheckExportedDatabase(addressesToVerify, cancellationToken); + } + + private async Task ExportDatabase(CancellationToken cancellationToken) + { + if (!FilePaths.TempDirExists()) Directory.CreateDirectory(FilePaths.PATH_TO_TEMP_DIR); + + await _dbExporter.ExportDb(FilePaths.PATH_TO_DUMP_FILE, cancellationToken); + } + + private async Task CheckExportedDatabase(List addressesToCheck, CancellationToken cancellationToken) + { + if (!FilePaths.DumpFileExists()) throw new FileNotFoundException($"The database dump file \"{FilePaths.PATH_TO_DUMP_FILE}\" doesn't exist."); + + var result = new DatabaseCheckResult + { + Success = true, + FoundOccurrences = [] + }; + + await foreach (var extractedTable in _sqlExtractor.ExtractTables(FilePaths.PATH_TO_DUMP_FILE).WithCancellation(cancellationToken)) + { + var found = addressesToCheck.ToDictionary(i => i, _ => 0); + + await foreach (var line in extractedTable.EntryLines.WithCancellation(cancellationToken)) + { + foreach (var identity in addressesToCheck) + { + var count = Regex.Matches(line, identity).Count; + found[identity] += count; + + if (count != 0) result.Success = false; + } + } + + if (found.Any(entry => entry.Value != 0)) + result.FoundOccurrences.Add(extractedTable.Id, found.Where(kvp => kvp.Value != 0).ToDictionary()); + } + + return result; + } + + public async Task SaveFoundOccurrences(DatabaseCheckResult result, CancellationToken cancellationToken) + { + var groupedOccurrences = result.FoundOccurrences + .GroupBy(x => x.Key.Schema) + .ToDictionary( + group => group.Key, + group => group.ToDictionary( + v => v.Key.Table, + v => v.Value + ) + ); + + await using var stream = new FileStream(FilePaths.PATH_TO_FOUND_FILE, FileMode.Create); + await JsonSerializer.SerializeAsync(stream, groupedOccurrences, SERIALIZER_OPTIONS, cancellationToken); + } +} + +public class DeletionFailedException(DatabaseCheckResult result) + : Exception($"Some identities were still found in the database ({result.NumberOfOccurrences} times in total). Check \"{FilePaths.PATH_TO_FOUND_FILE}\" for more details."); + +file static class FilePaths +{ + private const string DUMP_FILENAME = "db-dump.sql"; + private const string FOUND_FILENAME = "found-identities.json"; + + public static readonly string PATH_TO_TEMP_DIR = Path.Combine(Path.GetTempPath(), "enmeshed", "backbone"); + public static readonly string PATH_TO_DUMP_FILE = Path.Combine(PATH_TO_TEMP_DIR, DUMP_FILENAME); + public static readonly string PATH_TO_FOUND_FILE = Path.Combine(PATH_TO_TEMP_DIR, FOUND_FILENAME); + + public static bool TempDirExists() => Directory.Exists(PATH_TO_TEMP_DIR); + public static bool DumpFileExists() => File.Exists(PATH_TO_DUMP_FILE); +} diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/IDeletionVerifier.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/IDeletionVerifier.cs new file mode 100644 index 0000000000..685de179bf --- /dev/null +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/IDeletionVerifier.cs @@ -0,0 +1,20 @@ +using Backbone.DevelopmentKit.Identity.ValueObjects; +using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Extractors; + +namespace Backbone.Job.IdentityDeletion.IdentityDeletionVerifier; + +public interface IDeletionVerifier +{ + Task VerifyDeletion(List addressesToVerify, CancellationToken cancellationToken); + Task SaveFoundOccurrences(DatabaseCheckResult result, CancellationToken cancellationToken); +} + +public record DatabaseCheckResult +{ + public required bool Success { get; set; } + public required Dictionary> FoundOccurrences { get; init; } + + public int NumberOfOccurrences => FoundOccurrences + .SelectMany(v => v.Value.Values) + .Aggregate((a, b) => a + b); +} diff --git a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/DummyClasses/DummyDeletionVerifier.cs b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/DummyClasses/DummyDeletionVerifier.cs new file mode 100644 index 0000000000..214bee2a22 --- /dev/null +++ b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/DummyClasses/DummyDeletionVerifier.cs @@ -0,0 +1,29 @@ +using Backbone.DevelopmentKit.Identity.ValueObjects; +using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier; +using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Extractors; +using Backbone.Tooling.Extensions; + +namespace Backbone.Job.IdentityDeletion.Tests.Tests.DummyClasses; + +public class DummyDeletionVerifier : IDeletionVerifier +{ + private static readonly TableId TEST_ID = new() { Schema = "Test", Table = "Test" }; + + private readonly DatabaseCheckResult _result; + + public DummyDeletionVerifier(Dictionary occurrences) + { + Dictionary> found = []; + if (occurrences.Count != 0) + found[TEST_ID] = occurrences; + + _result = new DatabaseCheckResult + { + Success = occurrences.IsEmpty(), + FoundOccurrences = found + }; + } + + public Task VerifyDeletion(List addressesToVerify, CancellationToken cancellationToken) => Task.FromResult(_result); + public Task SaveFoundOccurrences(DatabaseCheckResult result, CancellationToken cancellationToken) => Task.CompletedTask; +} From 1148aa1b4b9eac0a0d62b55b2b5dde2643bc9a66 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 9 Sep 2025 14:05:02 +0200 Subject: [PATCH 21/51] chore: Add database configuration to identity deletion job configuration --- .../Job.IdentityDeletion/IdentityDeletionJobConfiguration.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionJobConfiguration.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionJobConfiguration.cs index 382bb26b4b..1cfa5fd2c1 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionJobConfiguration.cs +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionJobConfiguration.cs @@ -1,5 +1,6 @@ using System.ComponentModel.DataAnnotations; using Backbone.BuildingBlocks.Infrastructure.EventBus; +using Backbone.BuildingBlocks.Infrastructure.Persistence.Database; namespace Backbone.Job.IdentityDeletion; @@ -16,4 +17,7 @@ public class InfrastructureConfiguration { [Required] public required EventBusConfiguration EventBus { get; init; } + + [Required] + public required DatabaseConfiguration SqlDatabase { get; init; } } From 6936ef37797b93916524a4631e0efd41c315f652 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 9 Sep 2025 14:05:54 +0200 Subject: [PATCH 22/51] feat: Verify deletion after finishing --- .../Workers/ActualDeletionWorker.cs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs index bdeea19c27..b52a0635cf 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs @@ -1,6 +1,7 @@ using Backbone.BuildingBlocks.Application.Identities; using Backbone.BuildingBlocks.Application.PushNotifications; using Backbone.BuildingBlocks.Domain.Errors; +using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier; using Backbone.Modules.Devices.Application.Identities.Commands.HandleCompletedDeletionProcess; using Backbone.Modules.Devices.Application.Identities.Commands.HandleErrorDuringIdentityDeletion; using Backbone.Modules.Devices.Application.Identities.Commands.TriggerRipeDeletionProcesses; @@ -19,19 +20,22 @@ public class ActualDeletionWorker : IHostedService private readonly IPushNotificationSender _pushNotificationSender; private readonly ILogger _logger; private readonly List _identityDeleters; + private readonly IDeletionVerifier _deletionVerifier; public ActualDeletionWorker( IHostApplicationLifetime host, IEnumerable identityDeleters, IMediator mediator, IPushNotificationSender pushNotificationSender, - ILogger logger) + ILogger logger, + IDeletionVerifier deletionVerifier) { _host = host; _identityDeleters = identityDeleters.ToList(); _mediator = mediator; _pushNotificationSender = pushNotificationSender; _logger = logger; + _deletionVerifier = deletionVerifier; } public async Task StartAsync(CancellationToken cancellationToken) @@ -52,9 +56,16 @@ public async Task StartProcessing(CancellationToken cancellationToken) var addressesOfIdentitiesWithDeletionProcessesTriggeredInThePast = (await _mediator.Send(new ListAddressesOfIdentitiesWithDeletionProcessInStatusDeletingQuery(), cancellationToken)).Addresses; var addressesOfIdentitiesWithNewlyTriggeredDeletionProcesses = await TriggerRipeDeletionProcesses(cancellationToken); - var allAddressesToProcess = addressesOfIdentitiesWithDeletionProcessesTriggeredInThePast.Union(addressesOfIdentitiesWithNewlyTriggeredDeletionProcesses).Distinct(); + var allAddressesToProcess = addressesOfIdentitiesWithDeletionProcessesTriggeredInThePast.Union(addressesOfIdentitiesWithNewlyTriggeredDeletionProcesses).Distinct().ToList(); await Delete(allAddressesToProcess); + + var verifyResult = await _deletionVerifier.VerifyDeletion(allAddressesToProcess, cancellationToken); + if (!verifyResult.Success) + { + await _deletionVerifier.SaveFoundOccurrences(verifyResult, cancellationToken); + throw new DeletionFailedException(verifyResult); + } } private async Task> TriggerRipeDeletionProcesses(CancellationToken cancellationToken) From 631361c9b405471cc366c10ebc049e52e9c2babb Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 9 Sep 2025 14:06:27 +0200 Subject: [PATCH 23/51] chore: Add helper method to deconstruct connection string --- .../src/Job.IdentityDeletion/StringArrayExtensions.cs | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/StringArrayExtensions.cs diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/StringArrayExtensions.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/StringArrayExtensions.cs new file mode 100644 index 0000000000..8fadf71fd5 --- /dev/null +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/StringArrayExtensions.cs @@ -0,0 +1,6 @@ +namespace Backbone.Job.IdentityDeletion; + +public static class StringArrayExtensions +{ + public static string ExtractString(this string[] parts, string prefix) => parts.Single(x => x.StartsWith(prefix))[prefix.Length..]; +} From c5ad7aae44e1339a77107d9b62a954c8a0ab6795 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 9 Sep 2025 14:07:24 +0200 Subject: [PATCH 24/51] feat: Register deletion verifier, db exporter and sql extractor for dependency injection --- .../src/Job.IdentityDeletion/Program.cs | 5 ++++ .../ServicesExtensions.cs | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Program.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Program.cs index 8017983d47..ff8e2e47d0 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Program.cs +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Program.cs @@ -4,6 +4,7 @@ using Backbone.BuildingBlocks.Application.Identities; using Backbone.BuildingBlocks.Application.QuotaCheck; using Backbone.BuildingBlocks.Infrastructure.EventBus; +using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier; using Backbone.Modules.Announcements.Module; using Backbone.Modules.Challenges.Module; using Backbone.Modules.Devices.Module; @@ -108,6 +109,10 @@ public static IHostBuilder CreateHostBuilder(string[] args) services.RegisterIdentityDeleters(); + services.RegisterDbExporterAndExtractor(parsedConfiguration); + + services.AddTransient(); + services.AddEventBus(parsedConfiguration.Infrastructure.EventBus, METER_NAME); }) .UseServiceProviderFactory(new AutofacServiceProviderFactory()) diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/ServicesExtensions.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/ServicesExtensions.cs index 214b576b06..8ff5a3a094 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/ServicesExtensions.cs +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/ServicesExtensions.cs @@ -1,4 +1,8 @@ using Backbone.BuildingBlocks.Application.Identities; +using Backbone.BuildingBlocks.Infrastructure.Persistence.Database; +using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Exporters; +using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Extractors; +using PostgresSqlExtractor = Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Extractors.PostgresSqlExtractor; namespace Backbone.Job.IdentityDeletion; @@ -18,4 +22,25 @@ public static IServiceCollection RegisterIdentityDeleters(this IServiceCollectio return services; } + + public static IServiceCollection RegisterDbExporterAndExtractor(this IServiceCollection services, IdentityDeletionJobConfiguration parsedConfiguration) + { + switch (parsedConfiguration.Infrastructure.SqlDatabase.Provider) + { + case DatabaseConfiguration.POSTGRES: + services.AddTransient(); + services.AddTransient(); + break; + + case DatabaseConfiguration.SQLSERVER: + services.AddTransient(); + services.AddTransient(); + break; + + default: + throw new ArgumentOutOfRangeException($"No database provider registered for \"{parsedConfiguration.Infrastructure.SqlDatabase.Provider}\""); + } + + return services; + } } From d7caf1711114f451a5fd9b611bb213eb2334bd3b Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 9 Sep 2025 14:10:02 +0200 Subject: [PATCH 25/51] test: Add tests for the deletion verifier and the worker when the verifier fails --- .../Tests/ActualDeletionWorkerTests.cs | 42 ++++++- .../Tests/DeletionVerifierTests.cs | 108 ++++++++++++++++++ 2 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/DeletionVerifierTests.cs diff --git a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/ActualDeletionWorkerTests.cs b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/ActualDeletionWorkerTests.cs index d19688ac96..7f331558d4 100644 --- a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/ActualDeletionWorkerTests.cs +++ b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/ActualDeletionWorkerTests.cs @@ -2,6 +2,8 @@ using Backbone.BuildingBlocks.Application.PushNotifications; using Backbone.BuildingBlocks.Domain.Errors; using Backbone.DevelopmentKit.Identity.ValueObjects; +using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier; +using Backbone.Job.IdentityDeletion.Tests.Tests.DummyClasses; using Backbone.Job.IdentityDeletion.Workers; using Backbone.Modules.Devices.Application.Identities.Commands.TriggerRipeDeletionProcesses; using Backbone.Modules.Devices.Application.Identities.Queries.GetIdentity; @@ -97,6 +99,38 @@ public async Task Sends_push_notification_to_each_deleted_identity() } } + [Fact] + public async Task Not_deleted_identities_throw_an_error_when_verifying() + { + // Arrange + var fakeMediator = A.Fake(); + var identity1 = CreateIdentity(); + var identity2 = CreateIdentity(); + SetupRipeDeletionProcessesCommand(fakeMediator, identity1.Address, identity2.Address); + + A.CallTo(() => fakeMediator.Send(A._, A._)).Returns(new ListRelationshipsOfIdentityResponse([])); + + A.CallTo(() => fakeMediator.Send(A.That.Matches(q => q.Address == identity1.Address.Value), A._)) + .Returns(new GetIdentityResponse(identity1)); + + A.CallTo(() => fakeMediator.Send(A.That.Matches(q => q.Address == identity2.Address.Value), A._)) + .Returns(new GetIdentityResponse(identity2)); + + var notDeletedIdentities = new Dictionary + { + { identity1.Address.Value, 1 }, + { identity2.Address.Value, 2 } + }; + + var worker = CreateWorker(fakeMediator, null, null, notDeletedIdentities); + + // Act + var acting = async () => await worker.StartProcessing(CancellationToken.None); + + // Assert + await acting.ShouldThrowAsync(); + } + private static void SetupRipeDeletionProcessesCommand(IMediator mediator, params IdentityAddress[] identityAddresses) { var commandResponse = new TriggerRipeDeletionProcessesResponse(identityAddresses.ToDictionary(x => x.Value, _ => UnitResult.Success())); @@ -105,13 +139,17 @@ private static void SetupRipeDeletionProcessesCommand(IMediator mediator, params private static ActualDeletionWorker CreateWorker(IMediator mediator, List? identityDeleters = null, - IPushNotificationSender? pushNotificationSender = null) + IPushNotificationSender? pushNotificationSender = null, + Dictionary? notDeletedIdentities = null) { var hostApplicationLifetime = A.Dummy(); identityDeleters ??= [A.Dummy()]; pushNotificationSender ??= A.Dummy(); var logger = A.Dummy>(); - return new ActualDeletionWorker(hostApplicationLifetime, identityDeleters, mediator, pushNotificationSender, logger); + + notDeletedIdentities ??= []; + var dummyVerifier = new DummyDeletionVerifier(notDeletedIdentities); + return new ActualDeletionWorker(hostApplicationLifetime, identityDeleters, mediator, pushNotificationSender, logger, dummyVerifier); } private static Identity CreateIdentity() diff --git a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/DeletionVerifierTests.cs b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/DeletionVerifierTests.cs new file mode 100644 index 0000000000..2cc20831b8 --- /dev/null +++ b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests/Tests/DeletionVerifierTests.cs @@ -0,0 +1,108 @@ +using Backbone.DevelopmentKit.Identity.ValueObjects; +using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier; +using Backbone.Job.IdentityDeletion.Tests.Tests.DummyClasses; + +namespace Backbone.Job.IdentityDeletion.Tests.Tests; + +public class DeletionVerifierTests : AbstractTestsBase +{ + [Fact] + public async Task Empty_db_succeeds() + { + // Arrange + var identity = CreateRandomIdentityAddress(); + var deletionVerifier = CreateDeletionVerifier([]); + + // Act + var result = await deletionVerifier.VerifyDeletion([identity], CancellationToken.None); + + // Assert + result.Success.ShouldBeTrue(); + } + + [Fact] + public async Task Addresses_to_verify_not_found_succeeds() + { + // Arrange + List identitiesToVerify = [CreateRandomIdentityAddress(), CreateRandomIdentityAddress()]; + var deletionVerifier = CreateDeletionVerifier([ + Line.Single(CreateRandomIdentityAddress()), + Line.Concatenated(CreateRandomIdentityAddress(), CreateRandomIdentityAddress()), + Line.DataSingle(CreateRandomIdentityAddress()), + Line.DataConcatenated(CreateRandomIdentityAddress(), CreateRandomIdentityAddress()) + ]); + + // Act + var result = await deletionVerifier.VerifyDeletion(identitiesToVerify, CancellationToken.None); + + // Assert + result.Success.ShouldBeTrue(); + } + + [Fact] + public async Task A_simple_found_identity_to_verify_fails() + { + // Arrange + var identityToVerify = CreateRandomIdentityAddress(); + var deletionVerifier = CreateDeletionVerifier([ + Line.Single(identityToVerify) + ]); + + // Act + var result = await deletionVerifier.VerifyDeletion([identityToVerify], CancellationToken.None); + + // Assert + result.Success.ShouldBeFalse(); + result.NumberOfOccurrences.ShouldBe(1); + result.FoundOccurrences.ShouldContainKey(DummySqlExtractor.TEST_ID); + result.FoundOccurrences[DummySqlExtractor.TEST_ID].ShouldContainKeyAndValue(identityToVerify, 1); + } + + [Fact] + public async Task A_concatenated_found_identity_to_verify_fails() + { + // Arrange + var identityToVerify = CreateRandomIdentityAddress(); + var deletionVerifier = CreateDeletionVerifier([ + Line.Concatenated(identityToVerify, CreateRandomIdentityAddress()) + ]); + + // Act + var result = await deletionVerifier.VerifyDeletion([identityToVerify], CancellationToken.None); + + // Assert + result.Success.ShouldBeFalse(); + result.NumberOfOccurrences.ShouldBe(1); + result.FoundOccurrences.ShouldContainKey(DummySqlExtractor.TEST_ID); + result.FoundOccurrences[DummySqlExtractor.TEST_ID].ShouldContainKeyAndValue(identityToVerify, 1); + } + + [Fact] + public async Task A_found_identity_to_verify_in_a_data_string_fails() + { + // Arrange + var identityToVerify = CreateRandomIdentityAddress(); + var deletionVerifier = CreateDeletionVerifier([ + Line.DataSingle(identityToVerify) + ]); + + // Act + var result = await deletionVerifier.VerifyDeletion([identityToVerify], CancellationToken.None); + + // Assert + result.Success.ShouldBeFalse(); + result.NumberOfOccurrences.ShouldBe(1); + result.FoundOccurrences.ShouldContainKey(DummySqlExtractor.TEST_ID); + result.FoundOccurrences[DummySqlExtractor.TEST_ID].ShouldContainKeyAndValue(identityToVerify, 1); + } + + private static DeletionVerifier CreateDeletionVerifier(List lines) => new(new DummyDbExporter(), new DummySqlExtractor(lines)); +} + +file static class Line +{ + public static string Single(IdentityAddress address) => address.Value; + public static string Concatenated(IdentityAddress a, IdentityAddress b) => $"{a.Value}{b.Value}"; + public static string DataSingle(IdentityAddress address) => $"0\tabc\t\\N\t\\\\x41414141\t{address.Value}\t0"; + public static string DataConcatenated(IdentityAddress a, IdentityAddress b) => $"0\tabc\t\\N\t\\\\x41414141\t{a.Value}{b.Value}\t0"; +} From ae149c82f54e9c42f65b035339184e1a7b5b026e Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 9 Sep 2025 14:11:33 +0200 Subject: [PATCH 26/51] chore: Remove old deletion verifier workflow --- .github/workflows/test.yml | 96 -------------------------------------- 1 file changed, 96 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 24efc60498..ed7c69054e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -357,99 +357,3 @@ jobs: run: dotnet restore /p:ContinuousIntegrationBuild=true ./Backbone.sln - name: Validate Licenses run: nuget-license --input ./Backbone.sln --allowed-license-types ./.ci/allowedLicenses.json --ignored-packages ./.ci/ignoredPackages.json --output table --error-only - - verify-identity-deletion: - name: Validate Identity deletion (on ${{matrix.database.type}}) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - database: - - type: postgres - env: - Database__Provider: Postgres - Database__ConnectionString: "Server=postgres;Database=enmeshed;User Id=devices;Password=Passw0rd;Port=5432" - - type: sqlserver - env: - Database__Provider: SqlServer - Database__ConnectionString: "Server=sqlserver;Database=enmeshed;User Id=devices;Password=Passw0rd;TrustServerCertificate=True" - needs: image-test-builds - steps: - - name: Checkout - uses: actions/checkout@v5 - - - name: Setup Dotnet - uses: actions/setup-dotnet@v4 - with: - dotnet-version: 9.x - - - name: Cache NuGet packages - uses: actions/cache@v4 - with: - path: ~/.nuget/packages - key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj') }} - restore-keys: ${{ runner.os }}-nuget- - - - name: Download cached Docker images - uses: actions/download-artifact@v5 - with: - path: /tmp - pattern: docker-* - merge-multiple: true - - - name: Load Docker images and build applications - run: | - { - ./.ci/loadDockerImages.sh - } & - { - # The following two lines are for the identity deletion job only - mv appsettings.override.json appsettings.override.json.bak - cp .ci/appsettings.override.${{matrix.database.type}}.local.json appsettings.override.json - - dotnet restore ./Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj - dotnet restore ./Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Job.IdentityDeletion.csproj - dotnet restore ./Applications/AdminCli/src/AdminCli/AdminCli.csproj - - dotnet build --no-restore ./Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj - dotnet build --no-restore ./Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Job.IdentityDeletion.csproj - dotnet build --no-restore ./Applications/AdminCli/src/AdminCli/AdminCli.csproj - } - wait - - - name: Start compose stack - run: | - docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database.type}}.yml down -v - docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database.type}}.yml up --no-build --wait -d - docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database.type}}.yml wait admin-cli - - - name: Create identities, relationships etc. and initiate deletion processes - run: dotnet run --no-build --no-restore --project ./Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj init --consumerBaseUrl http://localhost:5000 --adminBaseUrl http://localhost:5173 - - - name: Run Identity deletion job - run: dotnet run --no-build --no-restore --project ./Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Job.IdentityDeletion.csproj --Worker ActualDeletionWorker - - - name: Export databse via Admin Cli - run: dotnet run --no-build --no-restore --project ./Applications/AdminCli/src/AdminCli/AdminCli.csproj database export --sensitive - env: ${{matrix.database.env}} - - - name: Check exported database file - run: dotnet run --no-build --no-restore --project ./Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj check - - - name: Save Docker Logs - if: failure() - run: docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database.type}}.yml logs > logs.txt - - - name: Archive logs - if: failure() - uses: actions/upload-artifact@v4 - with: - name: identity-deletion-verifier-docker-${{matrix.database.type}} - path: logs.txt - - - name: Archive exported database and identity files - if: failure() - uses: actions/upload-artifact@v4 - with: - name: identity-deletion-verifier-exported-db-${{matrix.database.type}} - path: /tmp/enmeshed/backbone From dbaa330bcd922fbd767cb8b77591011b4be24b49 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Wed, 10 Sep 2025 11:34:19 +0200 Subject: [PATCH 27/51] chore: Remove unneccessary file check --- .../IdentityDeletionVerifier/DeletionVerifier.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/DeletionVerifier.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/DeletionVerifier.cs index 2345570210..822925cadc 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/DeletionVerifier.cs +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/IdentityDeletionVerifier/DeletionVerifier.cs @@ -1,7 +1,5 @@ -using System.Text; -using System.Text.Json; +using System.Text.Json; using System.Text.RegularExpressions; -using Backbone.DevelopmentKit.Identity.ValueObjects; using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Exporters; using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier.Extractors; @@ -41,8 +39,6 @@ private async Task ExportDatabase(CancellationToken cancellationToken) private async Task CheckExportedDatabase(List addressesToCheck, CancellationToken cancellationToken) { - if (!FilePaths.DumpFileExists()) throw new FileNotFoundException($"The database dump file \"{FilePaths.PATH_TO_DUMP_FILE}\" doesn't exist."); - var result = new DatabaseCheckResult { Success = true, @@ -101,5 +97,4 @@ file static class FilePaths public static readonly string PATH_TO_FOUND_FILE = Path.Combine(PATH_TO_TEMP_DIR, FOUND_FILENAME); public static bool TempDirExists() => Directory.Exists(PATH_TO_TEMP_DIR); - public static bool DumpFileExists() => File.Exists(PATH_TO_DUMP_FILE); } From 37a2cea1643239292bd9bc7a4642d1cb869ef79a Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Wed, 10 Sep 2025 11:48:02 +0200 Subject: [PATCH 28/51] chore: Install pg_dump in docker image --- .../IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile index e3e41d0342..a7ea7dbb95 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile @@ -89,6 +89,8 @@ FROM base AS final RUN apk add icu-libs ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=0 +RUN apk add postgresql-client + WORKDIR /app COPY --from=publish /app/publish . ENTRYPOINT ["dotnet", "Backbone.Job.IdentityDeletion.dll"] From df5e4055e00183077ab95373cb61e922e303731b Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Wed, 10 Sep 2025 13:23:14 +0200 Subject: [PATCH 29/51] fix: Typo in Postgres connection string --- .ci/appsettings.override.postgres.local.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/appsettings.override.postgres.local.json b/.ci/appsettings.override.postgres.local.json index e458eabc8a..3e4ca1dc3f 100644 --- a/.ci/appsettings.override.postgres.local.json +++ b/.ci/appsettings.override.postgres.local.json @@ -19,7 +19,7 @@ }, "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User Id=adminUi;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" + "ConnectionString": "User ID=adminUi;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" } }, "ModuleDefaults": { From ede58257da1e44e3da7dd153d98f6168787cac61 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Wed, 10 Sep 2025 13:50:55 +0200 Subject: [PATCH 30/51] chore: Change pg_dump version in docker image --- .../IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile index a7ea7dbb95..8bb41b9ea3 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile @@ -89,7 +89,7 @@ FROM base AS final RUN apk add icu-libs ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=0 -RUN apk add postgresql-client +RUN apk add postgresql17-client WORKDIR /app COPY --from=publish /app/publish . From 8271e56cb211c4ab889896afbf87c566a4f64906 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Thu, 11 Sep 2025 12:48:33 +0200 Subject: [PATCH 31/51] fix: Make User ID uppercase in postgres appsettings override json --- .ci/appsettings.override.postgres.docker.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/appsettings.override.postgres.docker.json b/.ci/appsettings.override.postgres.docker.json index e18ffc9489..0d9e0ff092 100644 --- a/.ci/appsettings.override.postgres.docker.json +++ b/.ci/appsettings.override.postgres.docker.json @@ -19,7 +19,7 @@ }, "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User Id=adminUi;Password=Passw0rd;Server=postgres;Port=5432;Database=enmeshed;" + "ConnectionString": "User ID=adminUi;Password=Passw0rd;Server=postgres;Port=5432;Database=enmeshed;" } }, "ModuleDefaults": { From 9fd59bc19ac8f2719c789f4418e4f0bc84b9ca25 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Fri, 12 Sep 2025 11:03:52 +0200 Subject: [PATCH 32/51] test: Update apk package list before installing --- .../IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile index 8bb41b9ea3..5327ffaf7b 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile @@ -86,6 +86,8 @@ RUN dotnet publish /p:ContinuousIntegrationBuild=true -c Release --output /app/p FROM base AS final +RUN apk update + RUN apk add icu-libs ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=0 From 6985b8679a745151cb6795a612b4925c2eea26ef Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Fri, 12 Sep 2025 11:28:59 +0200 Subject: [PATCH 33/51] test: Log pg_dump version after installing --- .../IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile index 5327ffaf7b..203d151bef 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile @@ -93,6 +93,8 @@ ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=0 RUN apk add postgresql17-client +RUN pg_dump --version + WORKDIR /app COPY --from=publish /app/publish . ENTRYPOINT ["dotnet", "Backbone.Job.IdentityDeletion.dll"] From 6ec6c26031363778cc8e208b3912d58c3f7be476 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Fri, 12 Sep 2025 11:47:28 +0200 Subject: [PATCH 34/51] test: Log pg dump version on program startup --- .../IdentityDeletionJobs/src/Job.IdentityDeletion/Program.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Program.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Program.cs index ff8e2e47d0..ac5be4a017 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Program.cs +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Program.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using System.Reflection; using Autofac.Extensions.DependencyInjection; using Backbone.BuildingBlocks.API.Extensions; @@ -34,6 +35,9 @@ public static async Task Main(params string[] args) .WriteTo.Console() .CreateBootstrapLogger(); + var process = Process.Start("pg_dump", "--version"); + await process.WaitForExitAsync(); + try { Log.Information("Creating app..."); From 49d2efc7d8d99b5d8a68ec1e516f6fbae5fb1890 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Fri, 12 Sep 2025 11:55:19 +0200 Subject: [PATCH 35/51] test: Log pg dump version at the start of the worker --- .../src/Job.IdentityDeletion/Program.cs | 4 ---- .../Job.IdentityDeletion/Workers/ActualDeletionWorker.cs | 6 +++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Program.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Program.cs index ac5be4a017..ff8e2e47d0 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Program.cs +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Program.cs @@ -1,4 +1,3 @@ -using System.Diagnostics; using System.Reflection; using Autofac.Extensions.DependencyInjection; using Backbone.BuildingBlocks.API.Extensions; @@ -35,9 +34,6 @@ public static async Task Main(params string[] args) .WriteTo.Console() .CreateBootstrapLogger(); - var process = Process.Start("pg_dump", "--version"); - await process.WaitForExitAsync(); - try { Log.Information("Creating app..."); diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs index b52a0635cf..12a765fd08 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs @@ -1,4 +1,5 @@ -using Backbone.BuildingBlocks.Application.Identities; +using System.Diagnostics; +using Backbone.BuildingBlocks.Application.Identities; using Backbone.BuildingBlocks.Application.PushNotifications; using Backbone.BuildingBlocks.Domain.Errors; using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier; @@ -52,6 +53,9 @@ public Task StopAsync(CancellationToken cancellationToken) public async Task StartProcessing(CancellationToken cancellationToken) { + var process = Process.Start("pg_dump", "--version"); + await process.WaitForExitAsync(cancellationToken); + // In case there was an error during a previous run, we need to make sure we also process those identities again. var addressesOfIdentitiesWithDeletionProcessesTriggeredInThePast = (await _mediator.Send(new ListAddressesOfIdentitiesWithDeletionProcessInStatusDeletingQuery(), cancellationToken)).Addresses; var addressesOfIdentitiesWithNewlyTriggeredDeletionProcesses = await TriggerRipeDeletionProcesses(cancellationToken); From a73d51e4e267187e15f8c99474cbf2fa9bf87d6f Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Fri, 12 Sep 2025 12:13:43 +0200 Subject: [PATCH 36/51] test: Log pg dump version in integration test --- .../ActualDeletionWorkerTests.cs | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs index 3308f490eb..a80ff6d7ed 100644 --- a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs +++ b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs @@ -1,4 +1,5 @@ -using Backbone.DevelopmentKit.Identity.ValueObjects; +using System.Diagnostics; +using Backbone.DevelopmentKit.Identity.ValueObjects; using Backbone.Modules.Devices.Domain.Aggregates.Tier; using Backbone.Modules.Devices.Domain.Entities.Identities; using Backbone.Modules.Devices.Infrastructure.Persistence.Database; @@ -17,11 +18,13 @@ namespace Backbone.Job.IdentityDeletion.Tests.Integration; public class ActualDeletionWorkerTests : AbstractTestsBase { private readonly IHost _host; + private readonly ITestOutputHelper _testOutputHelper; - public ActualDeletionWorkerTests() + public ActualDeletionWorkerTests(ITestOutputHelper testOutputHelper) { var hostBuilder = Program.CreateHostBuilder(["--Worker", "ActualDeletionWorker"]); _host = hostBuilder.Build(); + _testOutputHelper = testOutputHelper; } [Fact] @@ -30,6 +33,19 @@ public async Task Logs_that_data_was_deleted() // Arrange var identity = await SeedDatabaseWithIdentityWithRipeDeletionProcess(); + var process = Process.Start(new ProcessStartInfo + { + FileName = "pg_dump", + UseShellExecute = false, + Arguments = "--version", + RedirectStandardOutput = true, + }); + + process.ShouldNotBeNull(); + + await process.WaitForExitAsync(TestContext.Current.CancellationToken); + _testOutputHelper.WriteLine(await process.StandardOutput.ReadToEndAsync(TestContext.Current.CancellationToken)); + // Act await _host.StartAsync(TestContext.Current.CancellationToken); From 2f7108a1394ccde22eaa036038b40aa385f86914 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Fri, 12 Sep 2025 12:41:12 +0200 Subject: [PATCH 37/51] test: Log pg dump path --- .../Workers/ActualDeletionWorker.cs | 7 ++- .../ActualDeletionWorkerTests.cs | 45 +++++++++++++------ 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs index 12a765fd08..5d5191d6b0 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs @@ -53,8 +53,11 @@ public Task StopAsync(CancellationToken cancellationToken) public async Task StartProcessing(CancellationToken cancellationToken) { - var process = Process.Start("pg_dump", "--version"); - await process.WaitForExitAsync(cancellationToken); + var versionProcess = Process.Start("pg_dump", "--version"); + await versionProcess.WaitForExitAsync(cancellationToken); + + var pathProcess = Process.Start("which", "pg_dump"); + await pathProcess.WaitForExitAsync(cancellationToken); // In case there was an error during a previous run, we need to make sure we also process those identities again. var addressesOfIdentitiesWithDeletionProcessesTriggeredInThePast = (await _mediator.Send(new ListAddressesOfIdentitiesWithDeletionProcessInStatusDeletingQuery(), cancellationToken)).Addresses; diff --git a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs index a80ff6d7ed..8b466ed412 100644 --- a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs +++ b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs @@ -32,19 +32,7 @@ public async Task Logs_that_data_was_deleted() { // Arrange var identity = await SeedDatabaseWithIdentityWithRipeDeletionProcess(); - - var process = Process.Start(new ProcessStartInfo - { - FileName = "pg_dump", - UseShellExecute = false, - Arguments = "--version", - RedirectStandardOutput = true, - }); - - process.ShouldNotBeNull(); - - await process.WaitForExitAsync(TestContext.Current.CancellationToken); - _testOutputHelper.WriteLine(await process.StandardOutput.ReadToEndAsync(TestContext.Current.CancellationToken)); + await LogPgDump(); // Act await _host.StartAsync(TestContext.Current.CancellationToken); @@ -84,6 +72,7 @@ public async Task Deletes_the_identity_when_it_is_in_status_ToBeDeleted() { // Arrange var identity = await SeedDatabaseWithIdentityWithRipeDeletionProcess(); + await LogPgDump(); // Act await _host.StartAsync(TestContext.Current.CancellationToken); @@ -100,6 +89,7 @@ public async Task Deletes_the_identity_when_it_is_in_status_Deleting() { // Arrange var identity = await SeedDatabaseWithIdentityInStatusDeleting(); + await LogPgDump(); // Act await _host.StartAsync(TestContext.Current.CancellationToken); @@ -119,6 +109,7 @@ public async Task Deletes_relationships() var peerOfIdentityToBeDeleted = await SeedDatabaseWithIdentity(); await SeedDatabaseWithActiveRelationshipBetween(identityToBeDeleted, peerOfIdentityToBeDeleted); + await LogPgDump(); // Act await _host.StartAsync(TestContext.Current.CancellationToken); @@ -137,6 +128,7 @@ public async Task Deletes_relationship_templates() var identityToBeDeleted = await SeedDatabaseWithIdentityWithRipeDeletionProcess(); await SeedDatabaseWithRelationshipTemplateOf(identityToBeDeleted.Address); + await LogPgDump(); // Act await _host.StartAsync(TestContext.Current.CancellationToken); @@ -225,5 +217,32 @@ private async Task SeedDatabaseWithIdentity() return identity; } + private async Task LogPgDump() + { + var versionProcess = Process.Start(new ProcessStartInfo + { + FileName = "pg_dump", + UseShellExecute = false, + Arguments = "--version", + RedirectStandardOutput = true, + }); + + versionProcess.ShouldNotBeNull(); + await versionProcess.WaitForExitAsync(TestContext.Current.CancellationToken); + _testOutputHelper.WriteLine(await versionProcess.StandardOutput.ReadToEndAsync(TestContext.Current.CancellationToken)); + + var pathProcess = Process.Start(new ProcessStartInfo + { + FileName = "which", + UseShellExecute = false, + Arguments = "pg_dump", + RedirectStandardOutput = true + }); + + pathProcess.ShouldNotBeNull(); + await pathProcess.WaitForExitAsync(TestContext.Current.CancellationToken); + _testOutputHelper.WriteLine(await pathProcess.StandardOutput.ReadToEndAsync(TestContext.Current.CancellationToken)); + } + #endregion } From ee2042fbee3c8d8900d9b563b6148f788de87ce7 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Mon, 15 Sep 2025 10:46:39 +0200 Subject: [PATCH 38/51] ci: Install Postgres 17 in pipeline --- .github/workflows/test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a58e163a69..ae36c0bd39 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -207,6 +207,14 @@ jobs: key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj') }} restore-keys: ${{ runner.os }}-nuget- + - name: Install Postgres 17 + run: | + sudo apt update + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' # Add Postgres APT Repo + curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg # Add GPG Key + sudo apt update + sudo apt install postgresql-17 + - name: Download cached Docker images uses: actions/download-artifact@v5 with: From 312ed0efcd27a5f1d404f66dc97250675707a932 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Mon, 15 Sep 2025 11:19:44 +0200 Subject: [PATCH 39/51] chore: Change Postgres User ID --- .ci/appsettings.override.postgres.docker.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/appsettings.override.postgres.docker.json b/.ci/appsettings.override.postgres.docker.json index 0d9e0ff092..fc29c9c40a 100644 --- a/.ci/appsettings.override.postgres.docker.json +++ b/.ci/appsettings.override.postgres.docker.json @@ -19,7 +19,7 @@ }, "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=adminUi;Password=Passw0rd;Server=postgres;Port=5432;Database=enmeshed;" + "ConnectionString": "User ID=postgres;Password=Passw0rd;Server=postgres;Port=5432;Database=enmeshed;" } }, "ModuleDefaults": { From 4b95189f42bfa05650b672b5282c1a2cb95007f4 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 16 Sep 2025 11:57:48 +0200 Subject: [PATCH 40/51] fix: Change User ID in local postgres appsettings override file --- .ci/appsettings.override.postgres.local.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/appsettings.override.postgres.local.json b/.ci/appsettings.override.postgres.local.json index 3e4ca1dc3f..8207b9aa94 100644 --- a/.ci/appsettings.override.postgres.local.json +++ b/.ci/appsettings.override.postgres.local.json @@ -19,7 +19,7 @@ }, "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=adminUi;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" + "ConnectionString": "User ID=postgres;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" } }, "ModuleDefaults": { From c6a1e39152ff4a1ac26a37fa14a9249f6ddcba20 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 16 Sep 2025 12:17:13 +0200 Subject: [PATCH 41/51] chore: Clean up unneccessary debug messages --- .../src/Job.IdentityDeletion/Dockerfile | 4 -- .../Workers/ActualDeletionWorker.cs | 9 +---- .../ActualDeletionWorkerTests.cs | 39 +------------------ 3 files changed, 3 insertions(+), 49 deletions(-) diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile index 203d151bef..8bb41b9ea3 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Dockerfile @@ -86,15 +86,11 @@ RUN dotnet publish /p:ContinuousIntegrationBuild=true -c Release --output /app/p FROM base AS final -RUN apk update - RUN apk add icu-libs ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=0 RUN apk add postgresql17-client -RUN pg_dump --version - WORKDIR /app COPY --from=publish /app/publish . ENTRYPOINT ["dotnet", "Backbone.Job.IdentityDeletion.dll"] diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs index 5d5191d6b0..b52a0635cf 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs @@ -1,5 +1,4 @@ -using System.Diagnostics; -using Backbone.BuildingBlocks.Application.Identities; +using Backbone.BuildingBlocks.Application.Identities; using Backbone.BuildingBlocks.Application.PushNotifications; using Backbone.BuildingBlocks.Domain.Errors; using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier; @@ -53,12 +52,6 @@ public Task StopAsync(CancellationToken cancellationToken) public async Task StartProcessing(CancellationToken cancellationToken) { - var versionProcess = Process.Start("pg_dump", "--version"); - await versionProcess.WaitForExitAsync(cancellationToken); - - var pathProcess = Process.Start("which", "pg_dump"); - await pathProcess.WaitForExitAsync(cancellationToken); - // In case there was an error during a previous run, we need to make sure we also process those identities again. var addressesOfIdentitiesWithDeletionProcessesTriggeredInThePast = (await _mediator.Send(new ListAddressesOfIdentitiesWithDeletionProcessInStatusDeletingQuery(), cancellationToken)).Addresses; var addressesOfIdentitiesWithNewlyTriggeredDeletionProcesses = await TriggerRipeDeletionProcesses(cancellationToken); diff --git a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs index e709ff794a..77ea89a470 100644 --- a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs +++ b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs @@ -1,5 +1,4 @@ -using System.Diagnostics; -using Backbone.DevelopmentKit.Identity.ValueObjects; +using Backbone.DevelopmentKit.Identity.ValueObjects; using Backbone.Modules.Devices.Domain.Aggregates.Tier; using Backbone.Modules.Devices.Domain.Entities.Identities; using Backbone.Modules.Devices.Infrastructure.Persistence.Database; @@ -18,13 +17,11 @@ namespace Backbone.Job.IdentityDeletion.Tests.Integration; public class ActualDeletionWorkerTests : AbstractTestsBase { private readonly IHost _host; - private readonly ITestOutputHelper _testOutputHelper; - public ActualDeletionWorkerTests(ITestOutputHelper testOutputHelper) + public ActualDeletionWorkerTests() { var hostBuilder = Program.CreateHostBuilder(["--Worker", "ActualDeletionWorker"]); _host = hostBuilder.Build(); - _testOutputHelper = testOutputHelper; } [Fact] @@ -32,7 +29,6 @@ public async Task Logs_that_data_was_deleted() { // Arrange var identity = await SeedDatabaseWithIdentityWithRipeDeletionProcess(); - await LogPgDump(); // Act await _host.StartAsync(TestContext.Current.CancellationToken); @@ -72,7 +68,6 @@ public async Task Deletes_the_identity_when_it_is_in_status_ToBeDeleted() { // Arrange var identity = await SeedDatabaseWithIdentityWithRipeDeletionProcess(); - await LogPgDump(); // Act await _host.StartAsync(TestContext.Current.CancellationToken); @@ -89,7 +84,6 @@ public async Task Deletes_the_identity_when_it_is_in_status_Deleting() { // Arrange var identity = await SeedDatabaseWithIdentityInStatusDeleting(); - await LogPgDump(); // Act await _host.StartAsync(TestContext.Current.CancellationToken); @@ -109,7 +103,6 @@ public async Task Deletes_relationships() var peerOfIdentityToBeDeleted = await SeedDatabaseWithIdentity(); await SeedDatabaseWithActiveRelationshipBetween(identityToBeDeleted, peerOfIdentityToBeDeleted); - await LogPgDump(); // Act await _host.StartAsync(TestContext.Current.CancellationToken); @@ -128,7 +121,6 @@ public async Task Deletes_relationship_templates() var identityToBeDeleted = await SeedDatabaseWithIdentityWithRipeDeletionProcess(); await SeedDatabaseWithRelationshipTemplateOf(identityToBeDeleted.Address); - await LogPgDump(); // Act await _host.StartAsync(TestContext.Current.CancellationToken); @@ -217,32 +209,5 @@ private async Task SeedDatabaseWithIdentity() return identity; } - private async Task LogPgDump() - { - var versionProcess = Process.Start(new ProcessStartInfo - { - FileName = "pg_dump", - UseShellExecute = false, - Arguments = "--version", - RedirectStandardOutput = true, - }); - - versionProcess.ShouldNotBeNull(); - await versionProcess.WaitForExitAsync(TestContext.Current.CancellationToken); - _testOutputHelper.WriteLine(await versionProcess.StandardOutput.ReadToEndAsync(TestContext.Current.CancellationToken)); - - var pathProcess = Process.Start(new ProcessStartInfo - { - FileName = "which", - UseShellExecute = false, - Arguments = "pg_dump", - RedirectStandardOutput = true - }); - - pathProcess.ShouldNotBeNull(); - await pathProcess.WaitForExitAsync(TestContext.Current.CancellationToken); - _testOutputHelper.WriteLine(await pathProcess.StandardOutput.ReadToEndAsync(TestContext.Current.CancellationToken)); - } - #endregion } From 8101f046632db40be1a0bf51601fc74b7b29679b Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Wed, 17 Sep 2025 11:41:45 +0200 Subject: [PATCH 42/51] test: Include passed integration tests in summary --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ae36c0bd39..25c29412a8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -244,7 +244,7 @@ jobs: docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database}}.yml wait admin-cli - name: Run integration tests - run: dotnet test --no-restore --no-build --logger "GitHubActions;summary.includeNotFoundTests=false;summary.includeSkippedTests=false;summary.includePassedTests=false" ${{matrix.test-project.path}} + run: dotnet test --no-restore --no-build --logger "GitHubActions;summary.includeNotFoundTests=false;summary.includeSkippedTests=false;summary.includePassedTests=true" ${{matrix.test-project.path}} env: ADMIN_API_BASE_ADDRESS: "http://localhost:5173" CONSUMER_API_BASE_ADDRESS: "http://localhost:5000" From e5096b247deaef9d35e83d591337c32d1c30a7a9 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Wed, 17 Sep 2025 12:55:09 +0200 Subject: [PATCH 43/51] test: Set a timeout of 30 seconds per test to identify the faulty one --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 25c29412a8..77c402de5c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -244,7 +244,7 @@ jobs: docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database}}.yml wait admin-cli - name: Run integration tests - run: dotnet test --no-restore --no-build --logger "GitHubActions;summary.includeNotFoundTests=false;summary.includeSkippedTests=false;summary.includePassedTests=true" ${{matrix.test-project.path}} + run: dotnet test --no-restore --no-build --logger "GitHubActions;summary.includeNotFoundTests=false;summary.includeSkippedTests=false;summary.includePassedTests=true" --blame-hang-timeout 30s ${{matrix.test-project.path}} env: ADMIN_API_BASE_ADDRESS: "http://localhost:5173" CONSUMER_API_BASE_ADDRESS: "http://localhost:5000" From ca21c4c9d0996fd66aa98c46bd8f76d6561349aa Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Thu, 18 Sep 2025 12:09:14 +0200 Subject: [PATCH 44/51] test: Log every single step --- .../ActualDeletionWorkerTests.cs | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs index 77ea89a470..91b377be69 100644 --- a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs +++ b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs @@ -17,11 +17,13 @@ namespace Backbone.Job.IdentityDeletion.Tests.Integration; public class ActualDeletionWorkerTests : AbstractTestsBase { private readonly IHost _host; + private readonly ITestOutputHelper _testOutputHelper; - public ActualDeletionWorkerTests() + public ActualDeletionWorkerTests(ITestOutputHelper testOutputHelper) { var hostBuilder = Program.CreateHostBuilder(["--Worker", "ActualDeletionWorker"]); _host = hostBuilder.Build(); + _testOutputHelper = testOutputHelper; } [Fact] @@ -96,21 +98,22 @@ public async Task Deletes_the_identity_when_it_is_in_status_Deleting() } [Fact] - public async Task Deletes_relationships() + public async Task Deletes_relationships() //TODO: Check deadlock { // Arrange - var identityToBeDeleted = await SeedDatabaseWithIdentityWithRipeDeletionProcess(); - var peerOfIdentityToBeDeleted = await SeedDatabaseWithIdentity(); + var identityToBeDeleted = await LogTime(SeedDatabaseWithIdentityWithRipeDeletionProcess(), "Seed DB with identity with ripe deletion process"); + var peerOfIdentityToBeDeleted = await LogTime(SeedDatabaseWithIdentity(), "Seed DB with identity"); - await SeedDatabaseWithActiveRelationshipBetween(identityToBeDeleted, peerOfIdentityToBeDeleted); + await LogTime(SeedDatabaseWithActiveRelationshipBetween(identityToBeDeleted, peerOfIdentityToBeDeleted), "Seed DB with relationship"); // Act - await _host.StartAsync(TestContext.Current.CancellationToken); + await LogTime(_host.StartAsync(TestContext.Current.CancellationToken), "Run Deletion Job"); // Assert var assertionContext = GetService(); - var relationshipsAfterAct = await assertionContext.Relationships.Where(Relationship.HasParticipant(identityToBeDeleted.Address)).ToListAsync(TestContext.Current.CancellationToken); + var relationshipsAfterAct = await LogTime(assertionContext.Relationships.Where(Relationship.HasParticipant(identityToBeDeleted.Address)).ToListAsync(TestContext.Current.CancellationToken), + "Get relationships"); relationshipsAfterAct.ShouldBeEmpty(); } @@ -209,5 +212,30 @@ private async Task SeedDatabaseWithIdentity() return identity; } + //Test method + private async Task LogTime(Task task, string hint = "") + { + _testOutputHelper.WriteLine($"Starting \"{hint}\""); + var start = DateTime.UtcNow; + + var result = await task; + + var duration = DateTime.UtcNow - start; + _testOutputHelper.WriteLine($"Completed \"{hint}\" (took {duration.TotalSeconds} s)"); + + return result; + } + + private async Task LogTime(Task task, string hint = "") + { + _testOutputHelper.WriteLine($"Starting \"{hint}\""); + var start = DateTime.UtcNow; + + await task; + + var duration = DateTime.UtcNow - start; + _testOutputHelper.WriteLine($"Completed \"{hint}\" (took {duration.TotalSeconds} s)"); + } + #endregion } From cfaa695bd52daf61176c1c75f3c2e011ffc84eca Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Thu, 18 Sep 2025 12:41:37 +0200 Subject: [PATCH 45/51] test: Capture everything --- .../ActualDeletionWorkerTests.cs | 4 ++-- .../Job.IdentityDeletion.Tests.Integration/AssemblyInfo.cs | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs index 91b377be69..91c6b88b28 100644 --- a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs +++ b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs @@ -228,13 +228,13 @@ private async Task LogTime(Task task, string hint = "") private async Task LogTime(Task task, string hint = "") { - _testOutputHelper.WriteLine($"Starting \"{hint}\""); + await Console.Error.WriteLineAsync($"Starting \"{hint}\""); var start = DateTime.UtcNow; await task; var duration = DateTime.UtcNow - start; - _testOutputHelper.WriteLine($"Completed \"{hint}\" (took {duration.TotalSeconds} s)"); + await Console.Error.WriteLineAsync($"Completed \"{hint}\" (took {duration.TotalSeconds} s)"); } #endregion diff --git a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/AssemblyInfo.cs b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/AssemblyInfo.cs index 495fa706b6..fca3429f45 100644 --- a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/AssemblyInfo.cs +++ b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/AssemblyInfo.cs @@ -1 +1,3 @@ [assembly: Trait("Category", "Integration")] +[assembly: CaptureConsole] +[assembly: CaptureTrace] From ea7eec00b469abc8d6050189fc2fd6aaa383d68a Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Fri, 19 Sep 2025 10:05:42 +0200 Subject: [PATCH 46/51] test: Remove one capture to trigger new run --- .../test/Job.IdentityDeletion.Tests.Integration/AssemblyInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/AssemblyInfo.cs b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/AssemblyInfo.cs index fca3429f45..9bb2d79b1b 100644 --- a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/AssemblyInfo.cs +++ b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/AssemblyInfo.cs @@ -1,3 +1,3 @@ [assembly: Trait("Category", "Integration")] -[assembly: CaptureConsole] +//[assembly: CaptureConsole] [assembly: CaptureTrace] From 22da433944dcdf11d062fe8370b54d83313c4889 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Fri, 19 Sep 2025 10:58:30 +0200 Subject: [PATCH 47/51] test: Don't verify --- .../src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs index b52a0635cf..36a59ae09d 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs @@ -60,12 +60,12 @@ public async Task StartProcessing(CancellationToken cancellationToken) await Delete(allAddressesToProcess); - var verifyResult = await _deletionVerifier.VerifyDeletion(allAddressesToProcess, cancellationToken); + /*var verifyResult = await _deletionVerifier.VerifyDeletion(allAddressesToProcess, cancellationToken); if (!verifyResult.Success) { await _deletionVerifier.SaveFoundOccurrences(verifyResult, cancellationToken); throw new DeletionFailedException(verifyResult); - } + }*/ } private async Task> TriggerRipeDeletionProcesses(CancellationToken cancellationToken) From f362ec7bda31f279a008d8d05f7691f5e75f59d5 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Fri, 19 Sep 2025 11:34:21 +0200 Subject: [PATCH 48/51] test: Re-add verification --- .../src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs index 36a59ae09d..b52a0635cf 100644 --- a/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs +++ b/Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Workers/ActualDeletionWorker.cs @@ -60,12 +60,12 @@ public async Task StartProcessing(CancellationToken cancellationToken) await Delete(allAddressesToProcess); - /*var verifyResult = await _deletionVerifier.VerifyDeletion(allAddressesToProcess, cancellationToken); + var verifyResult = await _deletionVerifier.VerifyDeletion(allAddressesToProcess, cancellationToken); if (!verifyResult.Success) { await _deletionVerifier.SaveFoundOccurrences(verifyResult, cancellationToken); throw new DeletionFailedException(verifyResult); - }*/ + } } private async Task> TriggerRipeDeletionProcesses(CancellationToken cancellationToken) From 7af156ee7a125e996030850c5caf7cd9d6e3420f Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Fri, 19 Sep 2025 11:50:29 +0200 Subject: [PATCH 49/51] test: Write a separate verification test and comment out the faulty one --- .../ActualDeletionWorkerTests.cs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs index 91c6b88b28..a88c39cd1e 100644 --- a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs +++ b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs @@ -1,4 +1,5 @@ using Backbone.DevelopmentKit.Identity.ValueObjects; +using Backbone.Job.IdentityDeletion.IdentityDeletionVerifier; using Backbone.Modules.Devices.Domain.Aggregates.Tier; using Backbone.Modules.Devices.Domain.Entities.Identities; using Backbone.Modules.Devices.Infrastructure.Persistence.Database; @@ -97,7 +98,7 @@ public async Task Deletes_the_identity_when_it_is_in_status_Deleting() identityAfterAct.ShouldBeNull(); } - [Fact] + /*[Fact] public async Task Deletes_relationships() //TODO: Check deadlock { // Arrange @@ -115,7 +116,7 @@ public async Task Deletes_relationships() //TODO: Check deadlock var relationshipsAfterAct = await LogTime(assertionContext.Relationships.Where(Relationship.HasParticipant(identityToBeDeleted.Address)).ToListAsync(TestContext.Current.CancellationToken), "Get relationships"); relationshipsAfterAct.ShouldBeEmpty(); - } + }*/ [Fact] public async Task Deletes_relationship_templates() @@ -135,6 +136,20 @@ public async Task Deletes_relationship_templates() templatesAfterAct.ShouldBeEmpty(); } + [Fact] + public async Task Verifies_deletion() + { + // Arrange + var identity = await LogTime(SeedDatabaseWithIdentity(), "Create Identity"); + var verifier = GetService(); + + // Act + var result = await LogTime(verifier.VerifyDeletion([identity.Address.Value], TestContext.Current.CancellationToken), "Verify Identity"); + + // Await + result.Success.ShouldBeFalse(); + } + private T GetService() where T : notnull { return _host.Services.CreateScope().ServiceProvider.GetRequiredService(); From 5d67d6bbf708ff0b521ebf6778c079cb3af8342a Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 23 Sep 2025 10:04:52 +0200 Subject: [PATCH 50/51] chore: Re-add faulty test, increase timeout to 60s --- .github/workflows/test.yml | 2 +- .../ActualDeletionWorkerTests.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 77c402de5c..9330162929 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -244,7 +244,7 @@ jobs: docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database}}.yml wait admin-cli - name: Run integration tests - run: dotnet test --no-restore --no-build --logger "GitHubActions;summary.includeNotFoundTests=false;summary.includeSkippedTests=false;summary.includePassedTests=true" --blame-hang-timeout 30s ${{matrix.test-project.path}} + run: dotnet test --no-restore --no-build --logger "GitHubActions;summary.includeNotFoundTests=false;summary.includeSkippedTests=false;summary.includePassedTests=true" --blame-hang-timeout 60s ${{matrix.test-project.path}} env: ADMIN_API_BASE_ADDRESS: "http://localhost:5173" CONSUMER_API_BASE_ADDRESS: "http://localhost:5000" diff --git a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs index a88c39cd1e..23c59ed055 100644 --- a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs +++ b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs @@ -98,7 +98,7 @@ public async Task Deletes_the_identity_when_it_is_in_status_Deleting() identityAfterAct.ShouldBeNull(); } - /*[Fact] + [Fact] public async Task Deletes_relationships() //TODO: Check deadlock { // Arrange @@ -116,7 +116,7 @@ public async Task Deletes_relationships() //TODO: Check deadlock var relationshipsAfterAct = await LogTime(assertionContext.Relationships.Where(Relationship.HasParticipant(identityToBeDeleted.Address)).ToListAsync(TestContext.Current.CancellationToken), "Get relationships"); relationshipsAfterAct.ShouldBeEmpty(); - }*/ + } [Fact] public async Task Deletes_relationship_templates() From dceabe232f34cc5dc6a9d0408619f6f43b580e56 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Wed, 24 Sep 2025 10:51:42 +0200 Subject: [PATCH 51/51] test: Log time in every test --- .../ActualDeletionWorkerTests.cs | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs index 23c59ed055..161775b555 100644 --- a/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs +++ b/Applications/IdentityDeletionJobs/test/Job.IdentityDeletion.Tests.Integration/ActualDeletionWorkerTests.cs @@ -31,16 +31,16 @@ public ActualDeletionWorkerTests(ITestOutputHelper testOutputHelper) public async Task Logs_that_data_was_deleted() { // Arrange - var identity = await SeedDatabaseWithIdentityWithRipeDeletionProcess(); + var identity = await LogTime(SeedDatabaseWithIdentityWithRipeDeletionProcess(), "Seed DB"); // Act - await _host.StartAsync(TestContext.Current.CancellationToken); + await LogTime(_host.StartAsync(TestContext.Current.CancellationToken), "Run Deletion process"); // Assert var assertionContext = GetService(); - var auditLogEntries = await assertionContext.IdentityDeletionProcessAuditLogs.Where(a => a.IdentityAddressHash == Hasher.HashUtf8(identity.Address)) - .ToListAsync(TestContext.Current.CancellationToken); + var auditLogEntries = await LogTime(assertionContext.IdentityDeletionProcessAuditLogs.Where(a => a.IdentityAddressHash == Hasher.HashUtf8(identity.Address)) + .ToListAsync(TestContext.Current.CancellationToken), "Get audit log entries"); var auditLogEntriesForDeletedData = auditLogEntries.Where(e => e.MessageKey == MessageKey.DataDeleted).ToList(); @@ -70,15 +70,15 @@ public async Task Logs_that_data_was_deleted() public async Task Deletes_the_identity_when_it_is_in_status_ToBeDeleted() { // Arrange - var identity = await SeedDatabaseWithIdentityWithRipeDeletionProcess(); + var identity = await LogTime(SeedDatabaseWithIdentityWithRipeDeletionProcess(), "Seed DB"); // Act - await _host.StartAsync(TestContext.Current.CancellationToken); + await LogTime(_host.StartAsync(TestContext.Current.CancellationToken), "Run Deletion process"); // Assert var assertionContext = GetService(); - var identityAfterAct = await assertionContext.Identities.FirstOrDefaultAsync(i => i.Address == identity.Address, TestContext.Current.CancellationToken); + var identityAfterAct = await LogTime(assertionContext.Identities.FirstOrDefaultAsync(i => i.Address == identity.Address, TestContext.Current.CancellationToken), "Get identity after act"); identityAfterAct.ShouldBeNull(); } @@ -86,15 +86,15 @@ public async Task Deletes_the_identity_when_it_is_in_status_ToBeDeleted() public async Task Deletes_the_identity_when_it_is_in_status_Deleting() { // Arrange - var identity = await SeedDatabaseWithIdentityInStatusDeleting(); + var identity = await LogTime(SeedDatabaseWithIdentityInStatusDeleting(), "Seed DB"); // Act - await _host.StartAsync(TestContext.Current.CancellationToken); + await LogTime(_host.StartAsync(TestContext.Current.CancellationToken), "Run Deletion process"); // Assert var assertionContext = GetService(); - var identityAfterAct = await assertionContext.Identities.FirstOrDefaultAsync(i => i.Address == identity.Address, TestContext.Current.CancellationToken); + var identityAfterAct = await LogTime(assertionContext.Identities.FirstOrDefaultAsync(i => i.Address == identity.Address, TestContext.Current.CancellationToken), "Get identity after act"); identityAfterAct.ShouldBeNull(); } @@ -122,17 +122,18 @@ public async Task Deletes_relationships() //TODO: Check deadlock public async Task Deletes_relationship_templates() { // Arrange - var identityToBeDeleted = await SeedDatabaseWithIdentityWithRipeDeletionProcess(); + var identityToBeDeleted = await LogTime(SeedDatabaseWithIdentityWithRipeDeletionProcess(), "Seed DB with identity with ripe deletion process"); - await SeedDatabaseWithRelationshipTemplateOf(identityToBeDeleted.Address); + await LogTime(SeedDatabaseWithRelationshipTemplateOf(identityToBeDeleted.Address), "Seed DB with relationship template"); // Act - await _host.StartAsync(TestContext.Current.CancellationToken); + await LogTime(_host.StartAsync(TestContext.Current.CancellationToken), "Run Deletion Job"); // Assert var assertionContext = GetService(); - var templatesAfterAct = await assertionContext.RelationshipTemplates.Where(rt => rt.CreatedBy == identityToBeDeleted.Address).ToListAsync(TestContext.Current.CancellationToken); + var templatesAfterAct = await LogTime(assertionContext.RelationshipTemplates.Where(rt => rt.CreatedBy == identityToBeDeleted.Address).ToListAsync(TestContext.Current.CancellationToken), + "Get relationship templates"); templatesAfterAct.ShouldBeEmpty(); } @@ -243,13 +244,13 @@ private async Task LogTime(Task task, string hint = "") private async Task LogTime(Task task, string hint = "") { - await Console.Error.WriteLineAsync($"Starting \"{hint}\""); + _testOutputHelper.WriteLine($"Starting \"{hint}\""); var start = DateTime.UtcNow; await task; var duration = DateTime.UtcNow - start; - await Console.Error.WriteLineAsync($"Completed \"{hint}\" (took {duration.TotalSeconds} s)"); + _testOutputHelper.WriteLine($"Completed \"{hint}\" (took {duration.TotalSeconds} s)"); } #endregion