From 5bf491c1dc0387a428c96e54eb778284e7474e36 Mon Sep 17 00:00:00 2001 From: danielpavic Date: Mon, 12 Oct 2026 20:16:06 +0200 Subject: [PATCH 01/14] Read the run configuration from the MigrationContext instead of IOptions MigrationService took IOptions from the container and read the products and the CLI tools from it, while the repository settings came from the MigrationContext. A host that builds one context per run from pre-built options (RayMigrator Studio) therefore failed with "Sequence contains no matching element" as soon as the container's options did not list the product. The service now reads everything from the context of the current run, and the IOptions parameter is gone from its constructor. TemplateCache no longer depends on the options either: a host builds the cache once per process and validates each configuration against it with ValidateConfigurationAgainstTemplateCache. DirectModePipeline and the engine test host do that right after building the cache, so the CLI behaves as before. Breaking change for hosts that construct MigrationService or TemplateCache themselves (allowed before 1.0). --- CHANGELOG.md | 13 +++++++++ .../TemplateCache.cs | 23 +++++---------- .../DirectModePipeline.cs | 6 ++-- .../MigrationService.cs | 28 ++++++++----------- .../Infrastructure/EngineTestHost.cs | 6 ++-- .../P1_CliToolStdinEncodingTests.cs | 3 +- .../P1_TemplateExecutorEnvironmentIdTests.cs | 10 +++---- ...teExecutorRepositoryCreatedLoggingTests.cs | 3 +- 8 files changed, 46 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef2cfa2..7051cc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ RayMigrator follows Semantic Versioning where applicable. ## [Unreleased] +### Changed + +- `MigrationService` reads the configuration of the current run from its + `MigrationContext` instead of from `IOptions`, and + `TemplateCache` no longer takes the options: a host builds the cache once + per process and calls `ValidateConfigurationAgainstTemplateCache` for each + configuration it runs. A host that creates one context per run from + pre-built options (RayMigrator Studio) can now execute migrations; before, + the service read the products and the CLI tools from the options registered + in the container, so a context built from other options failed with + "Sequence contains no matching element". The CLI behaves as before. Breaking + for hosts that construct `MigrationService` or `TemplateCache` themselves. + ### Fixed - `RULE_7_3` (hardcoded credential) no longer fires for a connection string diff --git a/Raycoon.RayMigrator.Infrastructure/TemplateCache.cs b/Raycoon.RayMigrator.Infrastructure/TemplateCache.cs index 90b05df..22a9c4b 100644 --- a/Raycoon.RayMigrator.Infrastructure/TemplateCache.cs +++ b/Raycoon.RayMigrator.Infrastructure/TemplateCache.cs @@ -1,5 +1,4 @@ using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; using Raycoon.RayMigrator.Core.Extensions; using Raycoon.RayMigrator.Core.Configuration; using Raycoon.RayMigrator.Core.Configuration.Options; @@ -10,33 +9,25 @@ namespace Raycoon.RayMigrator.Core.Templates; public class TemplateCache { - private readonly RayMigratorOptions? _options; private readonly bool _revealSensitiveData; private readonly ILogger _logger; private Dictionary> _templateDictionary = null!; /// - /// Constructor. Decoupled from MigrationContext for API compatibility. - /// When validateConfiguration is true (default), validates that configured DatabaseTypes have matching templates. - /// Set to false when Products/Repository config is not yet known (Admin-DB mode). + /// Loads the SQL templates of every DataAccessLayer found below the application base directory. + /// The cache does not depend on a configuration: a host that runs several configurations in one + /// process shares one cache and calls with + /// the options of each run. /// - /// RayMigrator configuration options. Can be null when validation is deferred. /// Whether to include sensitive data in log output. /// - /// Whether to validate Products/Repository against loaded templates at construction time. - public TemplateCache(IOptions? options, bool revealSensitiveData, ILogger logger, bool validateConfiguration = true) + public TemplateCache(bool revealSensitiveData, ILogger logger) { - _options = options?.Value; _revealSensitiveData = revealSensitiveData; _logger = logger; Initialize(); - - if (validateConfiguration && _options != null) - { - ValidateConfigurationAgainstTemplateCache(_options); - } } #region PrivateMethods @@ -225,8 +216,8 @@ private void LoadTemplatesFromFilesystem(string dataAccessLayersPath, } /// - /// Validates the TemplateCache against the configured usage. Can be called on-demand - /// when Products/Repository configuration becomes available (e.g. from Admin-DB). + /// Validates that the Repository and every TargetGroup of name a DatabaseType + /// for which templates were loaded. Hosts call it once per configuration before the first run. /// /// The RayMigratorOptions to validate against loaded templates. public void ValidateConfigurationAgainstTemplateCache(RayMigratorOptions options) diff --git a/Raycoon.RayMigrator.Pipeline/DirectModePipeline.cs b/Raycoon.RayMigrator.Pipeline/DirectModePipeline.cs index cb2d92f..0202aaf 100644 --- a/Raycoon.RayMigrator.Pipeline/DirectModePipeline.cs +++ b/Raycoon.RayMigrator.Pipeline/DirectModePipeline.cs @@ -179,9 +179,11 @@ public static async Task ExecuteAsync( // Register TemplateCache services.AddSingleton(serviceProvider => { - var opts = serviceProvider.GetRequiredService>(); + var opts = serviceProvider.GetRequiredService>().Value; var tcLogger = serviceProvider.GetRequiredService>(); - return new TemplateCache(opts, consoleOptions.RevealSensitiveData, tcLogger); + var templateCache = new TemplateCache(consoleOptions.RevealSensitiveData, tcLogger); + templateCache.ValidateConfigurationAgainstTemplateCache(opts); + return templateCache; }); services.AddSingleton(); diff --git a/Raycoon.RayMigrator.Services/MigrationService.cs b/Raycoon.RayMigrator.Services/MigrationService.cs index 66a63ef..5f14615 100644 --- a/Raycoon.RayMigrator.Services/MigrationService.cs +++ b/Raycoon.RayMigrator.Services/MigrationService.cs @@ -3,7 +3,6 @@ using System.Text.Json; using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; using Raycoon.RayMigrator.Core; using Raycoon.RayMigrator.Core.Configuration; using Raycoon.RayMigrator.Core.Configuration.Enums; @@ -27,7 +26,6 @@ namespace Raycoon.RayMigrator.Services; public class MigrationService : IMigrationService { private readonly ILogger _logger; - private readonly IOptions _options; private readonly TemplateExecutor _templateExecutor; private readonly IMigrationContextAccessor _ctxAccessor; private readonly IServiceProvider _serviceProvider; @@ -35,14 +33,12 @@ public class MigrationService : IMigrationService public MigrationService( ILogger logger, - IOptions options, TemplateExecutor templateExecutor, IMigrationContextAccessor ctxAccessor, IServiceProvider serviceProvider, ICliToolExecutor cliToolExecutor) { _logger = logger; - _options = options; _templateExecutor = templateExecutor; _ctxAccessor = ctxAccessor; _serviceProvider = serviceProvider; @@ -101,7 +97,7 @@ public async Task MigrateUpAsync(MigrateUpRequest requ } // --- Phase 2: File Discovery & Preparation --- - var productOptions = _options.Value.Products!.First(p => p.Alias == request.ProductAlias); + var productOptions = _ctxAccessor.Current.RayMigratorOptions.Products!.First(p => p.Alias == request.ProductAlias); var migrationFiles = DiscoverAndPrepareMigrationFiles(productOptions, request.Environment); if (migrationFiles.Count == 0) @@ -1118,7 +1114,7 @@ public async Task MigrateDownAsync(MigrateDownRequest // --- Validate mode: validate rollback file existence and parseability --- if (!request.RunMode.ShouldReadRepository()) { - var productOpts = _options.Value.Products!.First(p => p.Alias == request.ProductAlias); + var productOpts = _ctxAccessor.Current.RayMigratorOptions.Products!.First(p => p.Alias == request.ProductAlias); var allFiles = DiscoverAndPrepareMigrationFiles(productOpts, request.Environment); // Filter files in releases AFTER target (same logic as normal rollback) @@ -1172,7 +1168,7 @@ public async Task MigrateDownAsync(MigrateDownRequest // Validate target group aliases early (before any repository operations) { - var productOptsForValidation = _options.Value.Products!.First(p => p.Alias == request.ProductAlias); + var productOptsForValidation = _ctxAccessor.Current.RayMigratorOptions.Products!.First(p => p.Alias == request.ProductAlias); ValidateTargetGroupAliases(request.TargetGroupAliases, productOptsForValidation.TargetGroups!); } @@ -1268,7 +1264,7 @@ public async Task MigrateDownAsync(MigrateDownRequest migrationsToRollback.Count, request.ProductAlias, request.TargetReleaseVersion); // --- Phase 3: Execute rollbacks --- - var productOptions = _options.Value.Products!.First(p => p.Alias == request.ProductAlias); + var productOptions = _ctxAccessor.Current.RayMigratorOptions.Products!.First(p => p.Alias == request.ProductAlias); // The atomic shared-connection path changes what a failed rollback leaves behind; name the targets it applies to (#27). if (request.RunMode.ShouldExecuteSql()) @@ -1386,7 +1382,7 @@ public async Task BaselineAsync(BaselineRequest request) await RepositoryMigrationRunInsertWithAutoFix(settingsJson); // --- Phase 2: File Discovery --- - var productOptions = _options.Value.Products!.First(p => p.Alias == request.ProductAlias); + var productOptions = _ctxAccessor.Current.RayMigratorOptions.Products!.First(p => p.Alias == request.ProductAlias); var migrationFiles = DiscoverAndPrepareMigrationFiles(productOptions, request.Environment); if (migrationFiles.Count == 0) @@ -2376,7 +2372,7 @@ private MigrationFileInfo ParseMigrationFile( // Extract release version and target group from path (moved up — needed for ShouldSkipBlockSplitting) relativePath.GetReleaseVersionAndTargetGroupAlias( - _options.Value, _ctxAccessor.Current.RayMigratorConsoleOptions.Product, + _ctxAccessor.Current.RayMigratorOptions, _ctxAccessor.Current.RayMigratorConsoleOptions.Product, out string releaseVersion, out string targetGroupAlias); // Skip block splitting when CLI tool execution is configured. @@ -2763,7 +2759,7 @@ private string GetBlockDelimiter(string relativePath, ProductOptions productOpti try { relativePath.GetReleaseVersionAndTargetGroupAlias( - _options.Value, _ctxAccessor.Current.RayMigratorConsoleOptions.Product, + _ctxAccessor.Current.RayMigratorOptions, _ctxAccessor.Current.RayMigratorConsoleOptions.Product, out _, out string targetGroupAlias); var targetGroup = productOptions.TargetGroups! @@ -2804,7 +2800,7 @@ private string GetBlockDelimiter(string relativePath, ProductOptions productOpti /// private CliToolOptions GetCliToolByAlias(string alias) { - var cliTools = _options.Value.CliTools; + var cliTools = _ctxAccessor.Current.RayMigratorOptions.CliTools; if (cliTools == null || cliTools.Count == 0) { throw new ConfigurationValidationException( @@ -2887,7 +2883,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( string? fileContent = null; if (cliToolOptions.InputModeEnum == CliToolInputMode.Stdin) { - var productOptions = _options.Value.Products!.First(p => p.Alias == _ctxAccessor.Current.RayMigratorConsoleOptions.Product); + var productOptions = _ctxAccessor.Current.RayMigratorOptions.Products!.First(p => p.Alias == _ctxAccessor.Current.RayMigratorConsoleOptions.Product); fileContent = ReadMigrationText(file.FullPath, GetFileEncoding(productOptions.MigrationFilesEncoding), productOptions.MigrationFilesEncoding ?? "UTF-8", file.FilenameWithRelativePath, $"Check MigrationFilesEncoding for product [{productOptions.Alias}] or re-save the file in the configured encoding."); @@ -4579,7 +4575,7 @@ public async Task ValidateHashAsync(ValidateHashRequest reques bool repositoryHasProduct = await InitializeRepositoryAsync(); // --- Phase 2: File Discovery --- - var productOptions = _options.Value.Products!.First(p => p.Alias == request.ProductAlias); + var productOptions = _ctxAccessor.Current.RayMigratorOptions.Products!.First(p => p.Alias == request.ProductAlias); var migrationFiles = DiscoverAndPrepareMigrationFiles( productOptions, _ctxAccessor.Current.RayMigratorConsoleOptions.Environment); @@ -4758,7 +4754,7 @@ public async Task UpdateHashAsync(UpdateHashRequest request) await InitializeRepositoryAsync(); // --- Phase 2: File Discovery --- - var productOptions = _options.Value.Products!.First(p => p.Alias == request.ProductAlias); + var productOptions = _ctxAccessor.Current.RayMigratorOptions.Products!.First(p => p.Alias == request.ProductAlias); var migrationFiles = DiscoverAndPrepareMigrationFiles( productOptions, _ctxAccessor.Current.RayMigratorConsoleOptions.Environment); @@ -4883,7 +4879,7 @@ public async Task GetStatusAsync(string productAlias) : new List(); // --- Phase 3: File discovery --- - var productOptions = _options.Value.Products!.First(p => p.Alias == productAlias); + var productOptions = _ctxAccessor.Current.RayMigratorOptions.Products!.First(p => p.Alias == productAlias); var migrationFiles = DiscoverAndPrepareMigrationFiles( productOptions, _ctxAccessor.Current.RayMigratorConsoleOptions.Environment); diff --git a/Raycoon.RayMigrator.Tests.Engine/Infrastructure/EngineTestHost.cs b/Raycoon.RayMigrator.Tests.Engine/Infrastructure/EngineTestHost.cs index c5dbc9c..3e08e84 100644 --- a/Raycoon.RayMigrator.Tests.Engine/Infrastructure/EngineTestHost.cs +++ b/Raycoon.RayMigrator.Tests.Engine/Infrastructure/EngineTestHost.cs @@ -144,9 +144,11 @@ public void Build( // Register TemplateCache with explicit revealSensitiveData parameter services.AddSingleton(serviceProvider => { - var opts = serviceProvider.GetRequiredService>(); + var opts = serviceProvider.GetRequiredService>().Value; var tcLogger = serviceProvider.GetRequiredService>(); - return new TemplateCache(opts, consoleOptions.RevealSensitiveData, tcLogger); + var templateCache = new TemplateCache(consoleOptions.RevealSensitiveData, tcLogger); + templateCache.ValidateConfigurationAgainstTemplateCache(opts); + return templateCache; }); services.AddSingleton(); diff --git a/Raycoon.RayMigrator.Tests.Unit/P1_CliToolStdinEncodingTests.cs b/Raycoon.RayMigrator.Tests.Unit/P1_CliToolStdinEncodingTests.cs index cb20511..df48604 100644 --- a/Raycoon.RayMigrator.Tests.Unit/P1_CliToolStdinEncodingTests.cs +++ b/Raycoon.RayMigrator.Tests.Unit/P1_CliToolStdinEncodingTests.cs @@ -1,6 +1,5 @@ using System.Reflection; using AwesomeAssertions; -using Microsoft.Extensions.Options; using NSubstitute; using Raycoon.RayMigrator.Core; using Raycoon.RayMigrator.Core.Configuration.Enums; @@ -35,10 +34,10 @@ private static (MigrationService Service, ICliToolExecutor Executor) CreateServi new ProductOptions(null) { Alias = "P", MigrationFilesRootDirectory = "/tmp", MigrationFilesEncoding = productEncoding } } }; - Set(service, "_options", Options.Create(options)); Set(service, "_cliToolExecutor", executor); var accessor = (IMigrationContextAccessor)Get(service, "_ctxAccessor"); + accessor.Current.RayMigratorOptions = options; accessor.Current.RayMigratorConsoleOptions = new RayMigratorConsoleOptions { Command = MigrationCommand.MigrateUp, Product = "P", Environment = "Dev", RunMode = MigrationRunMode.Migrate, diff --git a/Raycoon.RayMigrator.Tests.Unit/P1_TemplateExecutorEnvironmentIdTests.cs b/Raycoon.RayMigrator.Tests.Unit/P1_TemplateExecutorEnvironmentIdTests.cs index cb57bae..7c4c64e 100644 --- a/Raycoon.RayMigrator.Tests.Unit/P1_TemplateExecutorEnvironmentIdTests.cs +++ b/Raycoon.RayMigrator.Tests.Unit/P1_TemplateExecutorEnvironmentIdTests.cs @@ -2,7 +2,6 @@ using System.Runtime.CompilerServices; using AwesomeAssertions; using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; using NSubstitute; using Raycoon.RayMigrator.Core; using Raycoon.RayMigrator.Core.Configuration.Enums; @@ -59,7 +58,7 @@ private static (TemplateExecutor executor, IDal dal, Func get }); var ctx = BuildContext(environmentId, productId, runMode); - var templateCache = BuildRealTemplateCache(ctx.RayMigratorOptions); + var templateCache = BuildRealTemplateCache(); // TemplateExecutor has a public constructor var accessor = new SingletonMigrationContextAccessor { Current = ctx }; @@ -172,12 +171,11 @@ private static MigrationContext BuildContext(int environmentId, int productId, M /// /// Builds a real TemplateCache from the DataAccessLayers in the test output directory. - /// validateConfiguration=false so we don't need all products/targets to have matching templates. + /// The cache is not validated against the options, so not every product needs matching templates. /// - private static TemplateCache BuildRealTemplateCache(RayMigratorOptions options) + private static TemplateCache BuildRealTemplateCache() { - var opts = Options.Create(options); - return new TemplateCache(opts, false, NullLogger.Instance, validateConfiguration: false); + return new TemplateCache(false, NullLogger.Instance); } /// diff --git a/Raycoon.RayMigrator.Tests.Unit/P1_TemplateExecutorRepositoryCreatedLoggingTests.cs b/Raycoon.RayMigrator.Tests.Unit/P1_TemplateExecutorRepositoryCreatedLoggingTests.cs index 51387db..9c1213e 100644 --- a/Raycoon.RayMigrator.Tests.Unit/P1_TemplateExecutorRepositoryCreatedLoggingTests.cs +++ b/Raycoon.RayMigrator.Tests.Unit/P1_TemplateExecutorRepositoryCreatedLoggingTests.cs @@ -2,7 +2,6 @@ using AwesomeAssertions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; using NSubstitute; using Raycoon.RayMigrator.Core; using Raycoon.RayMigrator.Core.Configuration; @@ -113,7 +112,7 @@ private static (TemplateExecutor Executor, CapturingLogger Log .Returns(Task.FromResult(scalarResult)); var context = BuildContext(); - var templateCache = new TemplateCache(Options.Create(context.RayMigratorOptions), false, NullLogger.Instance, validateConfiguration: false); + var templateCache = new TemplateCache(false, NullLogger.Instance); var logger = new CapturingLogger(); var executor = new TemplateExecutor(templateCache, logger, new SingletonMigrationContextAccessor { Current = context }); From d731601c9273853626b19c2eeb2fdc1b68fe14a6 Mon Sep 17 00:00:00 2001 From: danielpavic Date: Mon, 12 Oct 2026 20:26:32 +0200 Subject: [PATCH 02/14] Make IMigrationContextFactory the entry point of a host for one run The factory existed but no production code used it: DirectModePipeline and the engine test host built the MigrationContext themselves and then repeated the same post-build steps (DAL properties, schema names, logging context), and a host such as RayMigrator Studio that used the factory got none of them. Pre-built options also skipped everything the options framework does for options bound from files, so null timeouts failed later with an InvalidOperationException and every engine used the SQL Server block delimiter. RayMigratorOptionsPreparer now gives options a host built in memory the same treatment: {ENV:VARIABLE} placeholders are resolved in every string of the options graph (a missing variable names the variable and the path), the product defaults are merged, the enum values are probed, the data annotations run and write their defaults back, and the rule catalog is applied. The factory calls it in Create, skips it in CreateFromValidatedOptions for the CLI, fills the DAL properties, checks the schema names, and makes the context current for the calling async flow. ProductAliasGuard carries the explained product lookup that the pipeline had inline, so hosts get the same message. The factory moves to Raycoon.RayMigrator.Services because it needs DalFactory, takes the accessor and a logger, and loses the version argument. The accessor is a singleton in both host modes. The pipeline and the engine test host resolve the context through the factory and drop their duplicated steps. Breaking change for hosts that used the old factory signature (allowed before 1.0). --- CHANGELOG.md | 22 +++ .../Replacer/EnvironmentVariableReplacer.cs | 124 ++++++++++++ .../Validation/OptionsPreparationResult.cs | 13 ++ .../Validation/ProductAliasGuard.cs | 54 +++++ .../Validation/RayMigratorOptionsPreparer.cs | 72 +++++++ .../IMigrationContextFactory.cs | 78 +++----- Raycoon.RayMigrator.Core/MigrationContext.cs | 3 +- .../Raycoon.RayMigrator.Core.csproj | 1 + .../DirectModePipeline.cs | 127 +----------- .../MigrationContextFactory.cs | 112 +++++++++++ .../ServiceCollectionExtensions.cs | 8 +- .../Infrastructure/EngineTestHost.cs | 57 +----- .../P1_MigrationContextFactoryTests.cs | 136 +++++++++++++ .../P1_MigrationContextGuardTests.cs | 22 ++- .../P1_RayMigratorOptionsPreparerTests.cs | 184 ++++++++++++++++++ 15 files changed, 785 insertions(+), 228 deletions(-) create mode 100644 Raycoon.RayMigrator.Core/Configuration/Validation/OptionsPreparationResult.cs create mode 100644 Raycoon.RayMigrator.Core/Configuration/Validation/ProductAliasGuard.cs create mode 100644 Raycoon.RayMigrator.Core/Configuration/Validation/RayMigratorOptionsPreparer.cs create mode 100644 Raycoon.RayMigrator.Services/MigrationContextFactory.cs create mode 100644 Raycoon.RayMigrator.Tests.Unit/P1_MigrationContextFactoryTests.cs create mode 100644 Raycoon.RayMigrator.Tests.Unit/P1_RayMigratorOptionsPreparerTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7051cc5..7b878c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,28 @@ RayMigrator follows Semantic Versioning where applicable. ### Changed +- `IMigrationContextFactory` is the entry point of a host for one run, the CLI + pipeline included. `Create` prepares options a host built in memory with the + new `RayMigratorOptionsPreparer`, which gives them what options bound from + the `appsettings*.json` files always got: `{ENV:VARIABLE}` placeholders are + resolved in every string of the options graph (a missing variable is an + error that names the variable and the path), the product defaults are + merged down, every enum-typed value is checked, the data annotations run + and write their documented defaults into null properties, and the rule + catalog is applied. `CreateFromValidatedOptions` skips the preparation for + options the options framework already validated. Both fill the + `DalSpecificPropertiesDictionary` for the repository, the product's targets + and the database logging, check the schema names, and make the context + current (`IMigrationContextAccessor` and `MigrationLoggingContext`). A host + that built the context itself got none of this: pre-built options kept null + timeouts that later failed with an `InvalidOperationException`, every + engine used the SQL Server block delimiter `GO`, and the Serilog enricher + saw no run properties. The factory now lives in `Raycoon.RayMigrator.Services`, + takes the accessor and a logger, and no longer takes a version argument (the + engine knows its own). The accessor is a singleton in both host modes; the + API accessor isolates per async flow through its `AsyncLocal` anyway. + `ProductAliasGuard` explains an unknown product alias with the same message + the CLI showed, now also for hosts. - `MigrationService` reads the configuration of the current run from its `MigrationContext` instead of from `IOptions`, and `TemplateCache` no longer takes the options: a host builds the cache once diff --git a/Raycoon.RayMigrator.Core/Configuration/Replacer/EnvironmentVariableReplacer.cs b/Raycoon.RayMigrator.Core/Configuration/Replacer/EnvironmentVariableReplacer.cs index f4c933b..adac7c3 100644 --- a/Raycoon.RayMigrator.Core/Configuration/Replacer/EnvironmentVariableReplacer.cs +++ b/Raycoon.RayMigrator.Core/Configuration/Replacer/EnvironmentVariableReplacer.cs @@ -1,5 +1,9 @@ +using System.Collections; +using System.Reflection; using System.Text.RegularExpressions; using Microsoft.Extensions.Configuration; +using Raycoon.RayMigrator.Core.Configuration.Options; +using Raycoon.RayMigrator.Shared.Constants; using Raycoon.RayMigrator.Core.Configuration.Validation; namespace Raycoon.RayMigrator.Core.Configuration.Replacer; @@ -47,6 +51,126 @@ public static List ReplaceWithEnvironmentVariab return replacedEnvironmentVariables; } + /// + /// Replaces the {ENV:VARIABLE} placeholders in every string value of an options object in place: + /// string properties, string arrays and string dictionaries, recursively through the nested options classes + /// and their lists. The paths in the result follow the configuration keys (RayMigrator:Products:0:TargetGroups:1:Targets:0:ConnectionString). + /// A placeholder whose variable is not set is replaced by an empty string and reported with a null + /// ; the caller decides whether that is an error. + /// + public static List ReplaceWithEnvironmentVariables(RayMigratorOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var replaced = new List(); + ReplaceInObject(options, InternalConstants.RayMigratorSectionName, replaced); + return replaced; + } + + private static void ReplaceInObject(object target, string path, List replaced) + { + foreach (var property in target.GetType().GetProperties(BindingFlags.Public | BindingFlags.Instance)) + { + if (!property.CanRead || !property.CanWrite || property.GetIndexParameters().Length > 0) + { + continue; + } + + string propertyPath = $"{path}:{property.Name}"; + + if (property.PropertyType == typeof(string)) + { + var value = (string?)property.GetValue(target); + if (TryReplaceValue(value, propertyPath, replaced, out var replacedValue)) + { + property.SetValue(target, replacedValue); + } + + continue; + } + + if (property.PropertyType == typeof(string[])) + { + var array = (string[]?)property.GetValue(target); + for (int i = 0; array != null && i < array.Length; i++) + { + if (TryReplaceValue(array[i], $"{propertyPath}:{i}", replaced, out var replacedValue)) + { + array[i] = replacedValue!; + } + } + + continue; + } + + if (typeof(IDictionary).IsAssignableFrom(property.PropertyType)) + { + var dictionary = (IDictionary?)property.GetValue(target); + foreach (var key in dictionary?.Keys.ToList() ?? []) + { + if (TryReplaceValue(dictionary![key], $"{propertyPath}:{key}", replaced, out var replacedValue)) + { + dictionary[key] = replacedValue!; + } + } + + continue; + } + + if (IsOptionsClass(property.PropertyType)) + { + var nested = property.GetValue(target); + if (nested != null) + { + ReplaceInObject(nested, propertyPath, replaced); + } + + continue; + } + + if (property.GetValue(target) is IEnumerable list && property.PropertyType != typeof(string)) + { + int index = 0; + foreach (var item in list) + { + if (item != null && IsOptionsClass(item.GetType())) + { + ReplaceInObject(item, $"{propertyPath}:{index}", replaced); + } + + index++; + } + } + } + } + + private static bool IsOptionsClass(Type type) + => type.IsClass && type.Namespace == typeof(RayMigratorOptions).Namespace; + + private static bool TryReplaceValue(string? value, string path, List replaced, out string? replacedValue) + { + if (!TryReplaceStringContainingEnvironmentVariableReferences(value, out replacedValue, out var variableReplacements)) + { + return false; + } + + int separator = path.LastIndexOf(':'); + foreach (var variableReplacement in variableReplacements) + { + replaced.Add(new EnvironmentVariableWithMetadata + { + Path = path, + ConfigurationKey = separator < 0 ? path : path[(separator + 1)..], + ConfigurationValue = value, + ConfigurationValueReplaced = variableReplacement.ReplacedValue, + EnvironmentVariableName = variableReplacement.VariableName, + EnvironmentVariableValue = variableReplacement.VariableValue + }); + } + + return true; + } + /// /// Class for storing data for variable replacements. /// diff --git a/Raycoon.RayMigrator.Core/Configuration/Validation/OptionsPreparationResult.cs b/Raycoon.RayMigrator.Core/Configuration/Validation/OptionsPreparationResult.cs new file mode 100644 index 0000000..743b318 --- /dev/null +++ b/Raycoon.RayMigrator.Core/Configuration/Validation/OptionsPreparationResult.cs @@ -0,0 +1,13 @@ +using Raycoon.RayMigrator.Core.Configuration.Replacer; + +namespace Raycoon.RayMigrator.Core.Configuration.Validation; + +/// +/// What did to an options object. +/// +/// +/// Every {ENV:VARIABLE} placeholder that was replaced, with the configuration path it was found at. The +/// values are secrets as far as the engine is concerned; the preparer has registered them with +/// . +/// +public sealed record OptionsPreparationResult(IReadOnlyList ReplacedEnvironmentVariables); diff --git a/Raycoon.RayMigrator.Core/Configuration/Validation/ProductAliasGuard.cs b/Raycoon.RayMigrator.Core/Configuration/Validation/ProductAliasGuard.cs new file mode 100644 index 0000000..2261328 --- /dev/null +++ b/Raycoon.RayMigrator.Core/Configuration/Validation/ProductAliasGuard.cs @@ -0,0 +1,54 @@ +using Raycoon.RayMigrator.Core.Configuration.Options; +using Raycoon.RayMigrator.Shared.Exceptions; + +namespace Raycoon.RayMigrator.Core.Configuration.Validation; + +/// +/// Looks a product up by alias and explains a miss: the alias is case-sensitive, and a near miss or the list +/// of configured aliases is named in the message. +/// +public static class ProductAliasGuard +{ + /// + /// Returns the product with . + /// + /// No product carries that alias. + public static ProductOptions GetProduct(RayMigratorOptions options, string productAlias) + { + ArgumentNullException.ThrowIfNull(options); + + var configuredProducts = options.Products ?? []; + var match = configuredProducts.FirstOrDefault(p => p.Alias == productAlias); + if (match != null) + { + return match; + } + + var availableAliases = configuredProducts + .Where(p => !string.IsNullOrWhiteSpace(p.Alias)) + .Select(p => p.Alias!) + .ToList(); + + string message = $"Product alias [{productAlias}] not found in the loaded configuration."; + + var caseInsensitiveMatch = availableAliases + .FirstOrDefault(a => string.Equals(a, productAlias, StringComparison.OrdinalIgnoreCase)); + + if (caseInsensitiveMatch != null) + { + message += $"\nDid you mean [{caseInsensitiveMatch}]? The --product parameter is case-sensitive."; + } + else if (availableAliases.Count > 0) + { + message += $"\nAvailable product aliases: {string.Join(", ", availableAliases)}"; + message += "\nNote: The --product and --environment parameters are case-sensitive."; + } + else + { + message += "\nNo products are configured in the loaded configuration files."; + message += "\nNote: The --product and --environment parameters are case-sensitive."; + } + + throw new ConfigurationValidationException(message); + } +} diff --git a/Raycoon.RayMigrator.Core/Configuration/Validation/RayMigratorOptionsPreparer.cs b/Raycoon.RayMigrator.Core/Configuration/Validation/RayMigratorOptionsPreparer.cs new file mode 100644 index 0000000..ac95036 --- /dev/null +++ b/Raycoon.RayMigrator.Core/Configuration/Validation/RayMigratorOptionsPreparer.cs @@ -0,0 +1,72 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Options; +using Raycoon.RayMigrator.Core.Configuration.Options; +using Raycoon.RayMigrator.Core.Configuration.Replacer; +using Raycoon.RayMigrator.Shared.Exceptions; + +namespace Raycoon.RayMigrator.Core.Configuration.Validation; + +/// +/// Prepares a object that a host built in memory the way the CLI prepares the +/// options it binds from the appsettings*.json files: {ENV:VARIABLE} placeholders are resolved, +/// the product defaults are merged down, every enum-typed value is checked, the data annotations run (they also +/// write the documented default values back into null properties), and the rule catalog is applied. The object +/// is changed in place. Preparing an object twice is harmless. +/// +public static class RayMigratorOptionsPreparer +{ + /// + /// Prepares for a run. See the class summary for the steps. + /// + /// The options to prepare. Changed in place. + /// + /// The configuration as written, before placeholders were resolved, when the host has one. RULE_7_3 then + /// inspects the connection strings as written, so a credential that enters through a placeholder is not + /// reported as hardcoded (#30). Null checks the values as they are. + /// + /// + /// A placeholder names an environment variable that is not set or empty, an enum-typed value is not an + /// allowed member, a data annotation fails, or the rule catalog reports an error. The message lists every + /// finding of the failing step. + /// + public static OptionsPreparationResult Prepare(RayMigratorOptions options, IConfigurationSection? sectionAsWritten = null) + { + ArgumentNullException.ThrowIfNull(options); + + var replaced = EnvironmentVariableReplacer.ReplaceWithEnvironmentVariables(options); + var missing = replaced + .Where(r => string.IsNullOrWhiteSpace(r.EnvironmentVariableValue)) + .Select(r => $"[{r.EnvironmentVariableName}] at [{r.Path}]") + .Distinct() + .ToList(); + + if (missing.Count > 0) + { + throw new ConfigurationValidationException( + "The configuration references environment variables that do not exist, are empty or contain whitespace only: " + + string.Join(", ", missing)); + } + + SensitiveDataMasker.RegisterSensitiveValues(replaced.Select(r => r.EnvironmentVariableValue)); + + // MergeDefaults, then the enum probe that reports every unparseable value at once. + new ProductDefaultsPostConfigureOptions().PostConfigure(null, options); + + ThrowIfFailed(new DataAnnotationValidateOptions(null).Validate(null, options), "Configuration validation failed"); + ThrowIfFailed(new RayMigratorOptionsValidator(sectionAsWritten).Validate(null, options), "Configuration validation failed"); + + SensitiveDataMasker.RegisterSensitiveData(options); + + return new OptionsPreparationResult(replaced); + } + + private static void ThrowIfFailed(ValidateOptionsResult result, string prefix) + { + if (!result.Failed) + { + return; + } + + throw new ConfigurationValidationException($"{prefix}:{Environment.NewLine}{string.Join(Environment.NewLine, result.Failures ?? [])}"); + } +} diff --git a/Raycoon.RayMigrator.Core/IMigrationContextFactory.cs b/Raycoon.RayMigrator.Core/IMigrationContextFactory.cs index a15650f..e92399d 100644 --- a/Raycoon.RayMigrator.Core/IMigrationContextFactory.cs +++ b/Raycoon.RayMigrator.Core/IMigrationContextFactory.cs @@ -1,59 +1,43 @@ -using Raycoon.RayMigrator.Core.Configuration.Enums; using Raycoon.RayMigrator.Core.Configuration.Options; -using Raycoon.RayMigrator.Core.Configuration; +using Raycoon.RayMigrator.Core.Configuration.Validation; +using Raycoon.RayMigrator.Shared.Exceptions; namespace Raycoon.RayMigrator.Core; /// -/// Factory for creating MigrationContext instances. -/// CLI creates one at startup; API creates one per request. +/// Creates the of one run and makes it the current context. This is the entry +/// point for every host that runs the engine in-process, the CLI pipeline included. /// +/// +/// Both methods build the context, fill its for the +/// repository, the targets of the product and the database logging, check the schema names against the DAL +/// capabilities, and then set the and +/// of the calling async flow. The command in the console options is stamped into the MigrationRun settings +/// snapshot and drives the command's side effects (see ), so it must +/// be the command that is actually executed (#6). The run mode is only a choice for migrate-up and migrate-down; +/// every other command runs as . One context serves +/// one operation: its changes during the run. +/// public interface IMigrationContextFactory { /// - /// Creates a context for one command. The command is stamped into the MigrationRun settings snapshot and - /// drives the command's side effects (see ), so it must be the - /// command that is actually executed (#6). is only a choice for - /// / ; pass - /// for every other command. + /// Prepares with (placeholders, defaults, + /// enum check, data annotations, rule catalog) and creates the context. For options a host built in memory, + /// for example from a configuration store. /// - MigrationContext Create( - RayMigratorOptions options, - string product, - string environment, - MigrationCommand command, - MigrationRunMode runMode, - string version, - string? targetReleaseVersion = null, - bool revealSensitiveData = false); -} + /// + /// The options do not pass preparation, the product alias is unknown, a configured DatabaseType has no + /// data access layer, or a schema name is set for a database that does not support schemas. + /// + MigrationContext Create(RayMigratorOptions options, RayMigratorConsoleOptions consoleOptions); -/// -/// Default factory implementation that creates MigrationContext instances from parameters. -/// -public class MigrationContextFactory : IMigrationContextFactory -{ - public MigrationContext Create( - RayMigratorOptions options, - string product, - string environment, - MigrationCommand command, - MigrationRunMode runMode, - string version, - string? targetReleaseVersion = null, - bool revealSensitiveData = false) - { - var consoleOptions = new RayMigratorConsoleOptions - { - Command = command, - Product = product, - Environment = environment, - RunMode = runMode, - TargetReleaseVersion = targetReleaseVersion, - ShowStartupInfo = false, - RevealSensitiveData = revealSensitiveData, - }; - - return new MigrationContext(options, consoleOptions, version); - } + /// + /// Creates the context for options that the options framework already bound, merged and validated + /// (the CLI pipeline). Everything else is as in . + /// + /// + /// The product alias is unknown, a configured DatabaseType has no data access layer, or a schema name is set + /// for a database that does not support schemas. + /// + MigrationContext CreateFromValidatedOptions(RayMigratorOptions options, RayMigratorConsoleOptions consoleOptions); } diff --git a/Raycoon.RayMigrator.Core/MigrationContext.cs b/Raycoon.RayMigrator.Core/MigrationContext.cs index 36d5313..15cd23f 100644 --- a/Raycoon.RayMigrator.Core/MigrationContext.cs +++ b/Raycoon.RayMigrator.Core/MigrationContext.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using Raycoon.RayMigrator.Core.Configuration.Enums; using Raycoon.RayMigrator.Core.Configuration.Options; +using Raycoon.RayMigrator.Core.Configuration.Validation; using Raycoon.RayMigrator.Database.Common; using Raycoon.RayMigrator.Shared.Exceptions; @@ -38,7 +39,7 @@ public MigrationContext(RayMigratorOptions rayMigratorOptions, RayMigratorConsol RayMigratorConsoleOptions = rayMigratorConsoleOptions; RayMigratorVersion = rayMigratorVersion; - ProductTargetGroupOptionsEnumerable = RayMigratorOptions.Products!.First(p => p.Alias == RayMigratorConsoleOptions.Product).TargetGroups; + ProductTargetGroupOptionsEnumerable = ProductAliasGuard.GetProduct(RayMigratorOptions, RayMigratorConsoleOptions.Product).TargetGroups; // Create or set MigrationState if (migrationState == null) diff --git a/Raycoon.RayMigrator.Core/Raycoon.RayMigrator.Core.csproj b/Raycoon.RayMigrator.Core/Raycoon.RayMigrator.Core.csproj index 1f5ffb1..0581988 100644 --- a/Raycoon.RayMigrator.Core/Raycoon.RayMigrator.Core.csproj +++ b/Raycoon.RayMigrator.Core/Raycoon.RayMigrator.Core.csproj @@ -25,6 +25,7 @@ + diff --git a/Raycoon.RayMigrator.Pipeline/DirectModePipeline.cs b/Raycoon.RayMigrator.Pipeline/DirectModePipeline.cs index 0202aaf..0151369 100644 --- a/Raycoon.RayMigrator.Pipeline/DirectModePipeline.cs +++ b/Raycoon.RayMigrator.Pipeline/DirectModePipeline.cs @@ -189,17 +189,13 @@ public static async Task ExecuteAsync( services.AddSingleton(); services.AddScoped(); - // Create MigrationContext and set it on the singleton accessor + // Create the MigrationContext through the factory: it fills the DAL properties, checks the + // schema names and makes the context current (accessor and logging context) services.AddSingleton(serviceProvider => { var opts = serviceProvider.GetRequiredService>().Value; - string rayMigratorVersion = Raycoon.RayMigrator.Shared.AssemblyInfoHelper.GetRayMigratorVersion(); - var migCtx = new MigrationContext(opts, consoleOptions, rayMigratorVersion); - - var accessor = serviceProvider.GetRequiredService(); - accessor.Current = migCtx; - - return migCtx; + var factory = serviceProvider.GetRequiredService(); + return factory.CreateFromValidatedOptions(opts, consoleOptions); }); }) .Build(); @@ -224,49 +220,9 @@ public static async Task ExecuteAsync( RegisterSensitiveData(rayMigratorOptions); - #region Validate product alias - - var configuredProducts = rayMigratorOptions.Products ?? Enumerable.Empty(); - var matchingProduct = configuredProducts.FirstOrDefault(p => p.Alias == consoleOptions.Product); - - if (matchingProduct == null) - { - var availableAliases = configuredProducts - .Where(p => !string.IsNullOrWhiteSpace(p.Alias)) - .Select(p => p.Alias!) - .ToList(); - - string message = $"Product alias [{consoleOptions.Product}] not found in the loaded configuration."; - - var caseInsensitiveMatch = availableAliases - .FirstOrDefault(a => string.Equals(a, consoleOptions.Product, StringComparison.OrdinalIgnoreCase)); - - if (caseInsensitiveMatch != null) - { - message += $"\nDid you mean [{caseInsensitiveMatch}]? The --product parameter is case-sensitive."; - } - else if (availableAliases.Count > 0) - { - message += $"\nAvailable product aliases: {string.Join(", ", availableAliases)}"; - message += "\nNote: The --product and --environment parameters are case-sensitive."; - } - else - { - message += "\nNo products are configured in the loaded configuration files."; - message += "\nNote: The --product and --environment parameters are case-sensitive."; - } - - throw new ConfigurationValidationException(message); - } - - #endregion Validate product alias - - + // Resolving the context runs the factory: product lookup, DAL properties, schema names, accessor and logging context ctx = host.Services.GetRequiredService(); - // Set the ambient MigrationLoggingContext so the enricher can add migration properties to all log events - MigrationLoggingContext.Current = ctx; - // Resolve standard ILogger from DI (now backed purely by Serilog) logger = host.Services.GetRequiredService().CreateLogger(typeof(DirectModePipeline)); logger.LogDebug("Logger successfully created ({ModeName}). DatabaseLogWriter not yet initialized.", sourceResult.ModeName); @@ -307,46 +263,6 @@ public static async Task ExecuteAsync( #endregion Initialize DatabaseLogWriter - #region Initialize DalSpecificPropertiesDictionary - - logger.LogDebug(MigrationEvent.InitializeDalSpecificProperties, "Initializing DalSpecificPropertiesDictionary..."); - - SetDalSpecificPropertiesDictionary(ctx, rayMigratorOptions.Repository!.DatabaseType!, rayMigratorOptions.Repository!.ConnectionString!, logger); - - foreach (TargetGroupOptions targetGroupOptions in ctx.ProductTargetGroupOptionsEnumerable!) - { - foreach (var targetOptions in targetGroupOptions.Targets!) - { - SetDalSpecificPropertiesDictionary(ctx, targetGroupOptions.DatabaseType!, targetOptions.ConnectionString!, logger); - } - } - - // Add DatabaseLogging's DatabaseType to DalSpecificPropertiesDictionary (if configured) - if (rayMigratorOptions.DatabaseLogging?.DatabaseType != null) - { - SetDalSpecificPropertiesDictionary(ctx, rayMigratorOptions.DatabaseLogging.DatabaseType, - rayMigratorOptions.DatabaseLogging.ConnectionString!, logger); - } - - // Validate SchemaName based on DAL's SupportsSchema flag - SchemaNameValidator.ValidateSchemaName( - ctx.DalSpecificPropertiesDictionary, - rayMigratorOptions.Repository!.DatabaseType!, - rayMigratorOptions.Repository.SchemaName, - "Repository", logger); - - if (rayMigratorOptions.DatabaseLogging?.DatabaseType != null) - { - SchemaNameValidator.ValidateSchemaName( - ctx.DalSpecificPropertiesDictionary, - rayMigratorOptions.DatabaseLogging.DatabaseType, - rayMigratorOptions.DatabaseLogging.SchemaName, - "DatabaseLogging", logger); - } - - #endregion Initialize DalSpecificPropertiesDictionary - - #region Validate ConnectionStrings and DB-Connections logger.LogDebug(MigrationEvent.ValidateConnectionStrings, "Validating ConnectionStrings and DB-Connections..."); @@ -574,39 +490,6 @@ private static DatabaseLogWriter CreateDatabaseLogWriter( } } - /// - /// Sets the DalSpecificPropertiesDictionary from a configured DatabaseType. - /// - private static void SetDalSpecificPropertiesDictionary( - MigrationContext migrationContext, - string databaseType, - string connectionString, - Microsoft.Extensions.Logging.ILogger logger) - { - if (!DalFactory.TryGetDal(databaseType, connectionString, out IDal? dalInstance)) - { - throw new ConfigurationValidationException($"Could not retrieve data access layer for DatabaseType [{databaseType}]"); - } - - bool successfullyAdded = migrationContext.DalSpecificPropertiesDictionary.TryAdd(databaseType, dalInstance!.DalSpecificProperties); - if (successfullyAdded) - { - logger.LogDebug(MigrationEvent.InitializeDalSpecificProperties, - "DalSpecificProperties added for DatabaseType [{DatabaseType}] with MultiLineCommentStart: [{CommentStart}], MultiLineCommentEnd: [{CommentEnd}], SqlScriptDelimiter: [{Delimiter}], SupportsSchema: [{SupportsSchema}], SupportsTransactionalDdl: [{SupportsTransactionalDdl}]", - databaseType, - dalInstance.DalSpecificProperties.SqlMultiLineCommentStart, - dalInstance.DalSpecificProperties.SqlMultiLineCommentEnd, - string.IsNullOrWhiteSpace(dalInstance.DalSpecificProperties.SqlBlockDelimiter) ? "{Empty}" : dalInstance.DalSpecificProperties.SqlBlockDelimiter, - dalInstance.DalSpecificProperties.SupportsSchema, - dalInstance.DalSpecificProperties.SupportsTransactionalDdl); - - if (dalInstance is DalBase dalBase) - { - LogTransientErrorCodes(dalBase, databaseType, migrationContext.RayMigratorConsoleOptions.RevealSensitiveData, logger); - } - } - } - /// /// Reports where a DAL's transient error codes come from (ADR-022): Information when a TransientErrorCodes.txt /// replaced the built-in list (the path only with RevealSensitiveData), Debug for the built-in list, and a Warning diff --git a/Raycoon.RayMigrator.Services/MigrationContextFactory.cs b/Raycoon.RayMigrator.Services/MigrationContextFactory.cs new file mode 100644 index 0000000..835bee5 --- /dev/null +++ b/Raycoon.RayMigrator.Services/MigrationContextFactory.cs @@ -0,0 +1,112 @@ +using Microsoft.Extensions.Logging; +using Raycoon.RayMigrator.Core; +using Raycoon.RayMigrator.Core.Configuration; +using Raycoon.RayMigrator.Core.Configuration.Options; +using Raycoon.RayMigrator.Core.Configuration.Validation; +using Raycoon.RayMigrator.Core.Logging; +using Raycoon.RayMigrator.Database; +using Raycoon.RayMigrator.Database.Common; +using Raycoon.RayMigrator.Shared; +using Raycoon.RayMigrator.Shared.Exceptions; + +namespace Raycoon.RayMigrator.Services; + +/// +/// Default . Registered by . +/// +public sealed class MigrationContextFactory : IMigrationContextFactory +{ + private readonly IMigrationContextAccessor _ctxAccessor; + private readonly ILogger _logger; + + public MigrationContextFactory(IMigrationContextAccessor ctxAccessor, ILogger logger) + { + _ctxAccessor = ctxAccessor; + _logger = logger; + } + + /// + public MigrationContext Create(RayMigratorOptions options, RayMigratorConsoleOptions consoleOptions) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(consoleOptions); + + RayMigratorOptionsPreparer.Prepare(options); + + return Activate(new MigrationContext(options, consoleOptions, AssemblyInfoHelper.GetRayMigratorVersion())); + } + + /// + public MigrationContext CreateFromValidatedOptions(RayMigratorOptions options, RayMigratorConsoleOptions consoleOptions) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(consoleOptions); + + SensitiveDataMasker.RegisterSensitiveData(options); + + return Activate(new MigrationContext(options, consoleOptions, AssemblyInfoHelper.GetRayMigratorVersion())); + } + + private MigrationContext Activate(MigrationContext ctx) + { + var options = ctx.RayMigratorOptions; + + _logger.LogDebug(MigrationEvent.InitializeDalSpecificProperties, "Initializing DalSpecificPropertiesDictionary..."); + + SetDalSpecificProperties(ctx, options.Repository!.DatabaseType!, options.Repository.ConnectionString!); + + foreach (TargetGroupOptions targetGroupOptions in ctx.ProductTargetGroupOptionsEnumerable ?? []) + { + foreach (var targetOptions in targetGroupOptions.Targets ?? []) + { + SetDalSpecificProperties(ctx, targetGroupOptions.DatabaseType!, targetOptions.ConnectionString!); + } + } + + if (options.DatabaseLogging?.DatabaseType != null) + { + SetDalSpecificProperties(ctx, options.DatabaseLogging.DatabaseType, options.DatabaseLogging.ConnectionString!); + } + + SchemaNameValidator.ValidateSchemaName( + ctx.DalSpecificPropertiesDictionary, + options.Repository.DatabaseType!, + options.Repository.SchemaName, + "Repository", _logger); + + if (options.DatabaseLogging?.DatabaseType != null) + { + SchemaNameValidator.ValidateSchemaName( + ctx.DalSpecificPropertiesDictionary, + options.DatabaseLogging.DatabaseType, + options.DatabaseLogging.SchemaName, + "DatabaseLogging", _logger); + } + + _ctxAccessor.Current = ctx; + MigrationLoggingContext.Current = ctx; + + return ctx; + } + + private void SetDalSpecificProperties(MigrationContext ctx, string databaseType, string connectionString) + { + if (!DalFactory.TryGetDal(databaseType, connectionString, out IDal? dalInstance)) + { + throw new ConfigurationValidationException($"Could not retrieve data access layer for DatabaseType [{databaseType}]"); + } + + bool added = ctx.DalSpecificPropertiesDictionary.TryAdd(databaseType, dalInstance!.DalSpecificProperties); + if (added) + { + _logger.LogDebug(MigrationEvent.InitializeDalSpecificProperties, + "DalSpecificProperties added for DatabaseType [{DatabaseType}] with MultiLineCommentStart: [{CommentStart}], MultiLineCommentEnd: [{CommentEnd}], SqlScriptDelimiter: [{Delimiter}], SupportsSchema: [{SupportsSchema}], SupportsTransactionalDdl: [{SupportsTransactionalDdl}]", + databaseType, + dalInstance.DalSpecificProperties.SqlMultiLineCommentStart, + dalInstance.DalSpecificProperties.SqlMultiLineCommentEnd, + string.IsNullOrWhiteSpace(dalInstance.DalSpecificProperties.SqlBlockDelimiter) ? "{Empty}" : dalInstance.DalSpecificProperties.SqlBlockDelimiter, + dalInstance.DalSpecificProperties.SupportsSchema, + dalInstance.DalSpecificProperties.SupportsTransactionalDdl); + } + } +} diff --git a/Raycoon.RayMigrator.Services/ServiceCollectionExtensions.cs b/Raycoon.RayMigrator.Services/ServiceCollectionExtensions.cs index 0e20f8b..c4cd5d2 100644 --- a/Raycoon.RayMigrator.Services/ServiceCollectionExtensions.cs +++ b/Raycoon.RayMigrator.Services/ServiceCollectionExtensions.cs @@ -23,17 +23,19 @@ public static IServiceCollection AddRayMigratorServices(this IServiceCollection // Register CLI tool executor for external SQL tool execution services.AddScoped(); - // Register context accessor based on host mode + // Register context accessor based on host mode. Both are singletons: the CLI holds one context for the + // whole process, and the API accessor isolates per async flow through a static AsyncLocal, so a scoped + // registration would isolate nothing and would only make the singleton factory a captive dependency. if (hostMode == RayMigratorHostMode.Cli) { services.AddSingleton(); } else { - services.AddScoped(); + services.AddSingleton(); } - // Register context factory (always singleton, stateless) + // Register the context factory, the entry point of every host for one run services.AddSingleton(); return services; diff --git a/Raycoon.RayMigrator.Tests.Engine/Infrastructure/EngineTestHost.cs b/Raycoon.RayMigrator.Tests.Engine/Infrastructure/EngineTestHost.cs index 3e08e84..86efb5c 100644 --- a/Raycoon.RayMigrator.Tests.Engine/Infrastructure/EngineTestHost.cs +++ b/Raycoon.RayMigrator.Tests.Engine/Infrastructure/EngineTestHost.cs @@ -153,16 +153,11 @@ public void Build( services.AddSingleton(); - // Create MigrationContext and set it on the singleton accessor + // Create the MigrationContext through the factory (DAL properties, schema names, accessor, logging context) services.AddSingleton(serviceProvider => { - string rayMigratorVersion = AssemblyInfoHelper.GetRayMigratorVersion(); - var ctx = new MigrationContext(rayMigratorOptions, consoleOptions, rayMigratorVersion); - - var accessor = serviceProvider.GetRequiredService(); - accessor.Current = ctx; - - return ctx; + var factory = serviceProvider.GetRequiredService(); + return factory.CreateFromValidatedOptions(rayMigratorOptions, consoleOptions); }); }) .Build(); @@ -174,7 +169,6 @@ public void Build( var dbLogWriter = _dbLogWriter; var ctx = _host.Services.GetRequiredService(); - MigrationLoggingContext.Current = ctx; // Note: SensitiveDataMasker is initialized per-command in ScenarioContext methods // (BeginScope + RegisterSensitiveData), not here, to ensure AsyncLocal scope @@ -194,54 +188,11 @@ public void Build( } } - // Initialize DalSpecificPropertiesDictionary - SetDalSpecificPropertiesDictionary(ctx, rayMigratorOptions.Repository!.DatabaseType!, rayMigratorOptions.Repository!.ConnectionString!); - - foreach (TargetGroupOptions targetGroupOptions in ctx.ProductTargetGroupOptionsEnumerable!) - { - foreach (var targetOptions in targetGroupOptions.Targets!) - { - SetDalSpecificPropertiesDictionary(ctx, targetGroupOptions.DatabaseType!, targetOptions.ConnectionString!); - } - } - - // Add DatabaseLogging's DatabaseType to DalSpecificPropertiesDictionary (if configured) - if (rayMigratorOptions.DatabaseLogging?.DatabaseType != null) - { - SetDalSpecificPropertiesDictionary(ctx, rayMigratorOptions.DatabaseLogging.DatabaseType, - rayMigratorOptions.DatabaseLogging.ConnectionString!); - } - - // Validate SchemaName based on DAL's SupportsSchema flag - var logger = _host.Services.GetRequiredService>(); - - SchemaNameValidator.ValidateSchemaName( - ctx.DalSpecificPropertiesDictionary, - rayMigratorOptions.Repository!.DatabaseType!, - rayMigratorOptions.Repository.SchemaName, - "Repository", logger); - - if (rayMigratorOptions.DatabaseLogging?.DatabaseType != null) - { - SchemaNameValidator.ValidateSchemaName( - ctx.DalSpecificPropertiesDictionary, - rayMigratorOptions.DatabaseLogging.DatabaseType, - rayMigratorOptions.DatabaseLogging.SchemaName, - "DatabaseLogging", logger); - } - // Validate connections + var logger = _host.Services.GetRequiredService>(); ConnectionValidator.ValidateTargetConnections(ctx, logger); } - private static void SetDalSpecificPropertiesDictionary(MigrationContext ctx, string databaseType, string connectionString) - { - if (!DalFactory.TryGetDal(databaseType, connectionString, out IDal? dalInstance)) - return; - - ctx.DalSpecificPropertiesDictionary.TryAdd(databaseType, dalInstance!.DalSpecificProperties); - } - private static void ReplaceMigrationFilesRoot(IConfigurationSection section, string migrationFilesRoot) { foreach (var child in section.GetChildren()) diff --git a/Raycoon.RayMigrator.Tests.Unit/P1_MigrationContextFactoryTests.cs b/Raycoon.RayMigrator.Tests.Unit/P1_MigrationContextFactoryTests.cs new file mode 100644 index 0000000..45f9b0a --- /dev/null +++ b/Raycoon.RayMigrator.Tests.Unit/P1_MigrationContextFactoryTests.cs @@ -0,0 +1,136 @@ +using AwesomeAssertions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Raycoon.RayMigrator.Core; +using Raycoon.RayMigrator.Core.Configuration.Enums; +using Raycoon.RayMigrator.Core.Configuration.Options; +using Raycoon.RayMigrator.Services; +using Raycoon.RayMigrator.Shared.Exceptions; +using Raycoon.RayMigrator.Tests.Unit.Helpers; + +namespace Raycoon.RayMigrator.Tests.Unit; + +/// +/// P1: is the entry point of a host for one run: it prepares the options, +/// fills the DAL properties, and makes the context current for the calling async flow. +/// +[Collection("EnvironmentVariableTests")] +public class MigrationContextFactoryTests : IDisposable +{ + private readonly string _dir = Path.Combine(Path.GetTempPath(), "raymigrator-ctxf-" + Guid.NewGuid().ToString("N")); + private readonly List _variables = []; + + public MigrationContextFactoryTests() => Directory.CreateDirectory(_dir); + + public void Dispose() + { + foreach (var variable in _variables) + { + Environment.SetEnvironmentVariable(variable, null); + } + + MigrationLoggingContext.Current = null; + try { Directory.Delete(_dir, true); } catch { /* best effort */ } + } + + private static MigrationContextFactory CreateFactory(IMigrationContextAccessor? accessor = null) + => new(accessor ?? new SingletonMigrationContextAccessor(), NullLogger.Instance); + + private RayMigratorOptions CreateOptions(string targetConnectionString = "Data Source=:memory:", string? databaseLoggingConnectionString = null) + { + var options = ProfileTestContext.CreateOptions(targetConnectionString, databaseLoggingConnectionString); + options.Products![0].MigrationFilesRootDirectory = _dir; + return options; + } + + private static RayMigratorConsoleOptions ConsoleOptions(string product = ProfileTestContext.ProductAlias) + => new() + { + Command = MigrationCommand.MigrateUp, Product = product, Environment = "Docker", RunMode = MigrationRunMode.Simulate, + ShowStartupInfo = false, RevealSensitiveData = false, StopRollbackOnMissingRollbackFile = false + }; + + [Fact] + public void CreateFromValidatedOptions_FillsDalPropertiesForRepositoryTargetsAndLogging() + { + var options = CreateOptions(databaseLoggingConnectionString: "Data Source=:memory:"); + options.Products![0].TargetGroups!.Add(new TargetGroupOptions + { + Alias = "Pg", DatabaseType = "PostgreSQL", TargetMigrationOrder = "FileByFile", HashValidationScope = "File", + Targets = [new TargetOptions { Alias = "PgDb", ConnectionString = "Host=localhost;Database=raydb;Username=u;Password=p", DbCommandTimeoutInSeconds = 20, DbCommandMaxRetries = 0, DbCommandWaitTimeInMsBeforeRetry = 250 }] + }); + + var ctx = CreateFactory().CreateFromValidatedOptions(options, ConsoleOptions()); + + ctx.DalSpecificPropertiesDictionary.Keys.Should().BeEquivalentTo(["Sqlite", "PostgreSQL"], + "every DatabaseType of the repository, the product's targets and the database logging gets its DAL properties"); + ctx.DalSpecificPropertiesDictionary["PostgreSQL"].SqlBlockDelimiter.Should().NotBe("GO", + "a host that skipped this step fell back to the SQL Server delimiter for every engine"); + ctx.RayMigratorConsoleOptions.StopRollbackOnMissingRollbackFile.Should().BeFalse("the console options are taken as given"); + } + + [Fact] + public void Create_MakesTheContextCurrentForTheCallingFlow() + { + var accessor = new AsyncLocalMigrationContextAccessor(); + var factory = CreateFactory(accessor); + + var first = factory.Create(CreateOptions(), ConsoleOptions()); + accessor.Current.Should().BeSameAs(first); + MigrationLoggingContext.Current.Should().BeSameAs(first, "the Serilog enricher reads the ambient logging context"); + + var second = factory.Create(CreateOptions(), ConsoleOptions()); + accessor.Current.Should().BeSameAs(second, "a host creates one context per run and the newest one is current"); + MigrationLoggingContext.Current.Should().BeSameAs(second); + } + + [Fact] + public void Create_PreparesTheOptions() + { + string dirVar = "RM_CTXF_" + Guid.NewGuid().ToString("N")[..8].ToUpperInvariant(); + Environment.SetEnvironmentVariable(dirVar, _dir); + _variables.Add(dirVar); + var options = CreateOptions($"Data Source={{ENV:{dirVar}}}/target.db"); + options.Repository!.DbCommandTimeoutInSeconds = null; + + var ctx = CreateFactory().Create(options, ConsoleOptions()); + + ctx.RayMigratorOptions.Products![0].TargetGroups![0].Targets![0].ConnectionString.Should().Be($"Data Source={_dir}/target.db"); + ctx.RayMigratorOptions.Repository!.DbCommandTimeoutInSeconds.Should().Be(60); + } + + [Fact] + public void Create_UnknownProduct_ThrowsNamingTheAvailableAliases() + { + var act = () => CreateFactory().Create(CreateOptions(), ConsoleOptions(product: "Nope")); + + act.Should().Throw() + .WithMessage("*[Nope]*") + .WithMessage($"*{ProfileTestContext.ProductAlias}*"); + } + + [Fact] + public void Create_ProductAliasWithWrongCase_HintsAtTheConfiguredAlias() + { + var act = () => CreateFactory().Create(CreateOptions(), ConsoleOptions(product: ProfileTestContext.ProductAlias.ToUpperInvariant())); + + act.Should().Throw().WithMessage("*Did you mean*"); + } + + [Fact] + public void AddRayMigratorServices_ApiMode_ResolvesTheFactoryFromAScope() + { + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddRayMigratorServices(RayMigratorHostMode.Api); + // ValidateOnBuild is left off: TemplateCache and TemplateExecutor are registered by the host, not by AddRayMigratorServices. + using var provider = services.BuildServiceProvider(new ServiceProviderOptions { ValidateScopes = true }); + + using var scope = provider.CreateScope(); + var factory = scope.ServiceProvider.GetRequiredService(); + var ctx = factory.Create(CreateOptions(), ConsoleOptions()); + + scope.ServiceProvider.GetRequiredService().Current.Should().BeSameAs(ctx, + "the accessor is a singleton backed by an AsyncLocal, so the factory is not a captive dependency"); + } +} diff --git a/Raycoon.RayMigrator.Tests.Unit/P1_MigrationContextGuardTests.cs b/Raycoon.RayMigrator.Tests.Unit/P1_MigrationContextGuardTests.cs index 9828211..bbbac2e 100644 --- a/Raycoon.RayMigrator.Tests.Unit/P1_MigrationContextGuardTests.cs +++ b/Raycoon.RayMigrator.Tests.Unit/P1_MigrationContextGuardTests.cs @@ -1,6 +1,9 @@ using AwesomeAssertions; +using Microsoft.Extensions.Logging.Abstractions; using Raycoon.RayMigrator.Core; using Raycoon.RayMigrator.Core.Configuration.Enums; +using Raycoon.RayMigrator.Core.Configuration.Options; +using Raycoon.RayMigrator.Services; using Raycoon.RayMigrator.Shared.Exceptions; using Raycoon.RayMigrator.Tests.Unit.Helpers; @@ -60,7 +63,13 @@ public void Factory_StampsTheRequestedCommand(MigrationCommand command, Migratio { var options = ProfileTestContext.CreateOptions("Data Source=:memory:"); - var ctx = new MigrationContextFactory().Create(options, ProfileTestContext.ProductAlias, "Docker", command, runMode, "0.0.0-test", targetReleaseVersion: "1.0"); + var consoleOptions = new RayMigratorConsoleOptions + { + Command = command, Product = ProfileTestContext.ProductAlias, Environment = "Docker", RunMode = runMode, + TargetReleaseVersion = "1.0", ShowStartupInfo = false, RevealSensitiveData = false + }; + + var ctx = CreateFactory().CreateFromValidatedOptions(options, consoleOptions); ctx.RayMigratorConsoleOptions.Command.Should().Be(command, "a programmatic MigrateDown must not be recorded as MigrateUp"); ctx.RayMigratorConsoleOptions.RunMode.Should().Be(runMode); @@ -73,8 +82,17 @@ public void Factory_WithCommandNone_Throws() { var options = ProfileTestContext.CreateOptions("Data Source=:memory:"); - var act = () => new MigrationContextFactory().Create(options, ProfileTestContext.ProductAlias, "Docker", MigrationCommand.None, MigrationRunMode.Migrate, "0.0.0-test"); + var consoleOptions = new RayMigratorConsoleOptions + { + Command = MigrationCommand.None, Product = ProfileTestContext.ProductAlias, Environment = "Docker", + RunMode = MigrationRunMode.Migrate, ShowStartupInfo = false, RevealSensitiveData = false + }; + + var act = () => CreateFactory().CreateFromValidatedOptions(options, consoleOptions); act.Should().Throw(); } + + private static MigrationContextFactory CreateFactory() + => new(new SingletonMigrationContextAccessor(), NullLogger.Instance); } diff --git a/Raycoon.RayMigrator.Tests.Unit/P1_RayMigratorOptionsPreparerTests.cs b/Raycoon.RayMigrator.Tests.Unit/P1_RayMigratorOptionsPreparerTests.cs new file mode 100644 index 0000000..b25e419 --- /dev/null +++ b/Raycoon.RayMigrator.Tests.Unit/P1_RayMigratorOptionsPreparerTests.cs @@ -0,0 +1,184 @@ +using System.Text.Json; +using AwesomeAssertions; +using Raycoon.RayMigrator.Core.Configuration; +using Raycoon.RayMigrator.Core.Configuration.Options; +using Raycoon.RayMigrator.Core.Configuration.Replacer; +using Raycoon.RayMigrator.Core.Configuration.Validation; +using Raycoon.RayMigrator.Shared.Exceptions; +using Raycoon.RayMigrator.Tests.Unit.Helpers; + +namespace Raycoon.RayMigrator.Tests.Unit; + +/// +/// P1: gives options a host built in memory the same treatment the CLI +/// gives options bound from files: placeholders, defaults, enum probe, data annotations and the rule catalog. +/// +[Collection("EnvironmentVariableTests")] +public class RayMigratorOptionsPreparerTests : IDisposable +{ + private readonly string _dir = Path.Combine(Path.GetTempPath(), "raymigrator-prep-" + Guid.NewGuid().ToString("N")); + private readonly List _variables = []; + + public RayMigratorOptionsPreparerTests() => Directory.CreateDirectory(_dir); + + public void Dispose() + { + foreach (var variable in _variables) + { + Environment.SetEnvironmentVariable(variable, null); + } + + try { Directory.Delete(_dir, true); } catch { /* best effort */ } + } + + private string SetVariable(string value) + { + string name = "RM_PREP_" + Guid.NewGuid().ToString("N")[..8].ToUpperInvariant(); + Environment.SetEnvironmentVariable(name, value); + _variables.Add(name); + return name; + } + + private RayMigratorOptions CreateOptions(string targetConnectionString) + { + var options = ProfileTestContext.CreateOptions(targetConnectionString); + options.Products![0].MigrationFilesRootDirectory = _dir; + return options; + } + + [Fact] + public void ReplaceWithEnvironmentVariables_WalksStringsArraysDictionariesAndNestedLists() + { + string dirVar = SetVariable(_dir); + string toolVar = SetVariable("sqlite3"); + string hostVar = SetVariable("db.example.test"); + string codeVar = SetVariable("0"); + + var options = CreateOptions($"Data Source={{ENV:{dirVar}}}/target.db"); + options.Products![0].MigrationFilesRootDirectory = $"{{ENV:{dirVar}}}"; + options.Products[0].TargetGroups![0].Targets![0].CliToolParameters = new Dictionary { ["Host"] = $"{{ENV:{hostVar}}}" }; + options.CliTools = + [ + new CliToolOptions { Alias = "sqlite", ExecutablePath = $"{{ENV:{toolVar}}}", ArgumentTemplate = "{FilePath}", InputMode = "File", SuccessExitCodes = [$"{{ENV:{codeVar}}}"] } + ]; + + var replaced = EnvironmentVariableReplacer.ReplaceWithEnvironmentVariables(options); + + options.Products[0].MigrationFilesRootDirectory.Should().Be(_dir); + options.Products[0].TargetGroups![0].Targets![0].ConnectionString.Should().Be($"Data Source={_dir}/target.db"); + options.Products[0].TargetGroups![0].Targets![0].CliToolParameters!["Host"].Should().Be("db.example.test"); + options.CliTools[0].ExecutablePath.Should().Be("sqlite3"); + options.CliTools[0].SuccessExitCodes![0].Should().Be("0"); + + replaced.Select(r => r.Path).Should().BeEquivalentTo( + [ + "RayMigrator:Products:0:MigrationFilesRootDirectory", + "RayMigrator:Products:0:TargetGroups:0:Targets:0:ConnectionString", + "RayMigrator:Products:0:TargetGroups:0:Targets:0:CliToolParameters:Host", + "RayMigrator:CliTools:0:ExecutablePath", + "RayMigrator:CliTools:0:SuccessExitCodes:0", + ], "every replaced value is reported with the configuration path it was found at"); + replaced.Should().OnlyContain(r => r.EnvironmentVariableValue != null); + replaced.Single(r => r.Path.EndsWith("ExecutablePath")).EnvironmentVariableName.Should().Be(toolVar); + } + + [Fact] + public void ReplaceWithEnvironmentVariables_MissingVariable_ReportsNullValue() + { + var options = CreateOptions("Data Source={ENV:RM_PREP_DOES_NOT_EXIST}/target.db"); + + var replaced = EnvironmentVariableReplacer.ReplaceWithEnvironmentVariables(options); + + replaced.Should().ContainSingle().Which.EnvironmentVariableValue.Should().BeNull(); + options.Products![0].TargetGroups![0].Targets![0].ConnectionString.Should().Be("Data Source=/target.db", + "the placeholder is removed and the caller decides whether a missing variable is an error"); + } + + [Fact] + public void Prepare_ResolvesPlaceholdersAndRegistersTheValuesWithTheMasker() + { + using var scope = SensitiveDataMasker.BeginScope(revealSensitiveData: false); + string dirVar = SetVariable(_dir); + var options = CreateOptions($"Data Source={{ENV:{dirVar}}}/target.db"); + + var result = RayMigratorOptionsPreparer.Prepare(options); + + options.Products![0].TargetGroups![0].Targets![0].ConnectionString.Should().Be($"Data Source={_dir}/target.db"); + result.ReplacedEnvironmentVariables.Should().ContainSingle().Which.EnvironmentVariableName.Should().Be(dirVar); + SensitiveDataMasker.Mask($"path [{_dir}] and [{options.Repository!.ConnectionString}]") + .Should().NotContain(_dir, "a resolved environment variable value is a secret to the engine") + .And.NotContain("Data Source=:memory:", "the connection strings of the options are registered as well"); + } + + [Fact] + public void Prepare_MissingVariable_ThrowsNamingVariableAndPath() + { + var options = CreateOptions("Data Source={ENV:RM_PREP_DOES_NOT_EXIST}/target.db"); + + var act = () => RayMigratorOptionsPreparer.Prepare(options); + + act.Should().Throw() + .WithMessage("*RM_PREP_DOES_NOT_EXIST*") + .WithMessage("*RayMigrator:Products:0:TargetGroups:0:Targets:0:ConnectionString*"); + } + + [Fact] + public void Prepare_WritesTheDocumentedDefaultsIntoNullProperties() + { + var options = CreateOptions("Data Source=:memory:"); + options.Repository!.DbCommandTimeoutInSeconds = null; + options.Repository.DbCommandMaxRetries = null; + options.Repository.DbCommandWaitTimeInMsBeforeRetry = null; + var target = options.Products![0].TargetGroups![0].Targets![0]; + target.DbCommandTimeoutInSeconds = null; + target.DbCommandMaxRetries = null; + target.DbCommandWaitTimeInMsBeforeRetry = null; + + RayMigratorOptionsPreparer.Prepare(options); + + options.Repository.DbCommandTimeoutInSeconds.Should().Be(60, "the data annotation writes its default back, as it does for options bound from files"); + options.Repository.DbCommandMaxRetries.Should().Be(100); + options.Repository.DbCommandWaitTimeInMsBeforeRetry.Should().Be(250); + target.DbCommandTimeoutInSeconds.Should().Be(20, "the product defaults are merged down before the annotations run"); + target.DbCommandMaxRetries.Should().Be(0); + target.DbCommandWaitTimeInMsBeforeRetry.Should().Be(250); + } + + [Fact] + public void Prepare_InvalidEnumValue_Throws() + { + var options = CreateOptions("Data Source=:memory:"); + options.Products![0].MigrationErrorAction = "Explode"; + + var act = () => RayMigratorOptionsPreparer.Prepare(options); + + act.Should().Throw().WithMessage("*Explode*"); + } + + [Fact] + public void Prepare_RuleCatalogError_Throws() + { + var options = CreateOptions("Data Source=:memory:"); + var targets = options.Products![0].TargetGroups![0].Targets!; + targets.Add(new TargetOptions { Alias = targets[0].Alias, ConnectionString = "Data Source=:memory:" }); + + var act = () => RayMigratorOptionsPreparer.Prepare(options); + + act.Should().Throw().WithMessage($"*{ProfileTestContext.TargetAlias}*"); + } + + [Fact] + public void Prepare_Twice_LeavesTheOptionsUnchanged() + { + string dirVar = SetVariable(_dir); + var options = CreateOptions($"Data Source={{ENV:{dirVar}}}/target.db"); + options.Repository!.DbCommandTimeoutInSeconds = null; + + RayMigratorOptionsPreparer.Prepare(options); + string first = JsonSerializer.Serialize(options); + var second = RayMigratorOptionsPreparer.Prepare(options); + + JsonSerializer.Serialize(options).Should().Be(first); + second.ReplacedEnvironmentVariables.Should().BeEmpty("the placeholders were already resolved"); + } +} From e97a1c7533ba70c8eca5a04078d74f1a638d16d9 Mon Sep 17 00:00:00 2001 From: danielpavic Date: Mon, 12 Oct 2026 20:30:50 +0200 Subject: [PATCH 03/14] Stamp the release a run starts from into MigrationRun.FromReleaseVersion The parameter was bound to its own name instead of a value, so every run row stored the literal text "FromReleaseVersion". The run now records the highest Migrated release of the product and environment at the time it starts (null for a first run). The records are read in a dedicated query right before the run row is inserted, because the run row is also the run lock and the reads that drive the run must stay behind it. GetHighestMigratedRelease is now the one definition of that "current release": Info reports it, the out-of-order check quotes it, and the run insert stamps it. --- CHANGELOG.md | 5 ++ .../template-execution-order.md | 2 +- Raycoon.RayMigrator.Core/MigrationContext.cs | 1 + Raycoon.RayMigrator.Core/MigrationState.cs | 6 +++ .../TemplateExecutor.cs | 2 +- .../MigrationService.cs | 28 +++++++---- .../Infrastructure/ScenarioContext.cs | 5 ++ .../Features/SqliteMigrationRunMetaTests.cs | 20 ++++++++ .../P1_HighestMigratedReleaseTests.cs | 49 +++++++++++++++++++ .../P1_TemplateExecutorEnvironmentIdTests.cs | 29 ++++++++++- 10 files changed, 135 insertions(+), 12 deletions(-) create mode 100644 Raycoon.RayMigrator.Tests.Unit/P1_HighestMigratedReleaseTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b878c2..32c8019 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,11 @@ RayMigrator follows Semantic Versioning where applicable. ### Fixed +- `MigrationRun.FromReleaseVersion` holds the release the run started from: + the highest Migrated release of the product and environment, read right + before the run row is inserted, or `NULL` for a first run. Every run row + stored the literal text `FromReleaseVersion` before, because the parameter + was bound to its own name instead of a value. - `RULE_7_3` (hardcoded credential) no longer fires for a connection string that supplies its credential through an `{ENV:VARIABLE}` placeholder. The CLI ran the rule catalog on the resolved values, so a placeholder resolving diff --git a/Docs/03-database-layer/template-execution-order.md b/Docs/03-database-layer/template-execution-order.md index d1101d1..fde0635 100644 --- a/Docs/03-database-layer/template-execution-order.md +++ b/Docs/03-database-layer/template-execution-order.md @@ -263,7 +263,7 @@ Called immediately after Product registration by every `MigrationService` entry - `@MigrationRunModeId` - From console options `RunMode` (cast to byte) - `@MigratorMetaId` - From `MigrationState.MigratorMetaId` - `@MigrationRunResultId` - Set to `Running` (10) from `MigrationState.MigrationRunResult` -- `@FromReleaseVersion` - Currently hardcoded as `"FromReleaseVersion"` +- `@FromReleaseVersion` - From `MigrationState.FromReleaseVersion`: the highest Migrated release of the product and environment when the run started, read right before the insert; `NULL` when nothing was migrated yet - `@ToReleaseVersion` - From console options `TargetReleaseVersion` - `@MigrationRunSettingsJson` - JSON snapshot of all RayMigrator settings at migration start diff --git a/Raycoon.RayMigrator.Core/MigrationContext.cs b/Raycoon.RayMigrator.Core/MigrationContext.cs index 15cd23f..e63123e 100644 --- a/Raycoon.RayMigrator.Core/MigrationContext.cs +++ b/Raycoon.RayMigrator.Core/MigrationContext.cs @@ -60,6 +60,7 @@ public MigrationContext(RayMigratorOptions rayMigratorOptions, RayMigratorConsol FileOrderId = migrationState.FileOrderId, FileBlockId = migrationState.FileBlockId, FileBlocksCommitted = migrationState.FileBlocksCommitted, + FromReleaseVersion = migrationState.FromReleaseVersion, MigrationRunResult = migrationState.MigrationRunResult, MigrationOperation = migrationState.MigrationOperation, TargetGroupAlias = migrationState.TargetGroupAlias, diff --git a/Raycoon.RayMigrator.Core/MigrationState.cs b/Raycoon.RayMigrator.Core/MigrationState.cs index 66a8d31..5a82d8f 100644 --- a/Raycoon.RayMigrator.Core/MigrationState.cs +++ b/Raycoon.RayMigrator.Core/MigrationState.cs @@ -33,6 +33,12 @@ public class MigrationState /// public int FileBlocksCommitted { get; set; } + /// + /// The highest release (ordinal, case-insensitive) among the Migrated records of the product and environment + /// when the run started, or null when nothing was migrated yet. Stamped into MigrationRun.FromReleaseVersion. + /// + public string? FromReleaseVersion { get; set; } + // Migration Process: Step / Result public MigrationRunResult MigrationRunResult { get; set; } public MigrationOperation MigrationOperation { get; set; } diff --git a/Raycoon.RayMigrator.Infrastructure/TemplateExecutor.cs b/Raycoon.RayMigrator.Infrastructure/TemplateExecutor.cs index 4613257..0934f33 100644 --- a/Raycoon.RayMigrator.Infrastructure/TemplateExecutor.cs +++ b/Raycoon.RayMigrator.Infrastructure/TemplateExecutor.cs @@ -217,7 +217,7 @@ public void RepositoryMigrationRunInsert(string migrationRunSettingsJson) dalParameterList.AddParameter(new DalParameter("MigrationRunModeId", (byte) _ctxAccessor.Current.RayMigratorConsoleOptions.RunMode, typeof(byte))); dalParameterList.AddParameter(new DalParameter("MigratorMetaId", _ctxAccessor.Current.MigrationState.MigratorMetaId, typeof(int))); dalParameterList.AddParameter(new DalParameter("MigrationRunResultId", (byte) _ctxAccessor.Current.MigrationState.MigrationRunResult, typeof(byte))); - dalParameterList.AddParameter(new DalParameter("FromReleaseVersion", "FromReleaseVersion", typeof(string))); + dalParameterList.AddParameter(new DalParameter("FromReleaseVersion", _ctxAccessor.Current.MigrationState.FromReleaseVersion, typeof(string))); dalParameterList.AddParameter(new DalParameter("ToReleaseVersion", _ctxAccessor.Current.RayMigratorConsoleOptions.TargetReleaseVersion, typeof(string))); dalParameterList.AddParameter(new DalParameter("MigrationRunSettingsJson", migrationRunSettingsJson, typeof(string))); diff --git a/Raycoon.RayMigrator.Services/MigrationService.cs b/Raycoon.RayMigrator.Services/MigrationService.cs index 5f14615..5a2d3f0 100644 --- a/Raycoon.RayMigrator.Services/MigrationService.cs +++ b/Raycoon.RayMigrator.Services/MigrationService.cs @@ -171,11 +171,7 @@ public async Task MigrateUpAsync(MigrateUpRequest requ { if (!request.AllowOutOfOrder) { - string highestMigratedRelease = existingRecords - .Where(r => r.MigrationStatusId == MigrationStatus.Migrated) - .OrderByDescending(r => r.ReleaseVersion, StringComparer.OrdinalIgnoreCase) - .Select(r => r.ReleaseVersion) - .First(); + string highestMigratedRelease = GetHighestMigratedRelease(existingRecords)!; _logger.LogError( "Out-of-order migrations detected but --allow-out-of-order not specified. {Count} file(s) from releases before '{HighestRelease}' not applied. Aborting.", @@ -3523,6 +3519,18 @@ internal static MigrationOperation DeriveRunOperation(IEnumerable + /// The highest release (ordinal, case-insensitive) among the Migrated records, or null when none is migrated. + /// This is the "current release" of a product and environment: the Info command reports it, the out-of-order + /// check quotes it, and every run stamps it into MigrationRun.FromReleaseVersion. + /// + internal static string? GetHighestMigratedRelease(IEnumerable records) + => records + .Where(r => r.MigrationStatusId == MigrationStatus.Migrated) + .OrderByDescending(r => r.ReleaseVersion, StringComparer.OrdinalIgnoreCase) + .Select(r => r.ReleaseVersion) + .FirstOrDefault(); + /// /// Attempts to insert a MigrationRun, auto-fixing orphaned runs if a parallel-run lock is detected. /// When RepositoryMigrationRunInsert fails with MigrationAlreadyRunningException, this method @@ -3531,6 +3539,11 @@ internal static MigrationOperation DeriveRunOperation(IEnumerable internal async Task RepositoryMigrationRunInsertWithAutoFix(string settingsJson) { + // The release the run starts from: a dedicated read, because the run row is also the run lock and the + // reads that drive the run itself must happen after it. + var recordsBeforeRun = await Task.Run(() => _templateExecutor.RepositoryMigrationSelect()); + _ctxAccessor.Current.MigrationState.FromReleaseVersion = GetHighestMigratedRelease(recordsBeforeRun); + try { await Task.Run(() => _templateExecutor.RepositoryMigrationRunInsert(settingsJson)); @@ -4889,10 +4902,7 @@ public async Task GetStatusAsync(string productAlias) .ToList(); // Determine current release (highest release version among migrated records) - string currentRelease = migratedRecords - .OrderByDescending(r => r.ReleaseVersion, StringComparer.OrdinalIgnoreCase) - .Select(r => r.ReleaseVersion) - .FirstOrDefault() ?? "None"; + string currentRelease = GetHighestMigratedRelease(existingRecords) ?? "None"; // Count pending migrations: (file, target) pairs that are not yet successfully migrated. // Evaluated per target like migrate-up does, so a target that failed while another diff --git a/Raycoon.RayMigrator.Tests.Engine/Infrastructure/ScenarioContext.cs b/Raycoon.RayMigrator.Tests.Engine/Infrastructure/ScenarioContext.cs index 2d11921..a45bb0f 100644 --- a/Raycoon.RayMigrator.Tests.Engine/Infrastructure/ScenarioContext.cs +++ b/Raycoon.RayMigrator.Tests.Engine/Infrastructure/ScenarioContext.cs @@ -492,6 +492,11 @@ public void AssertTargetGroupMigrationOrder(params string[] expectedOrder) /// public string? GetMigrationRunSettingsJson() => _queryHelper.GetLatestMigrationRunSettingsJson(); + /// + /// Returns the MigrationRun row by index (1-based, oldest first), or null when there is no such run. + /// + public MigrationRunRecordDto? GetMigrationRun(int runIndex) => _queryHelper.GetMigrationRunByIndex(runIndex); + /// /// Gets the FileUpConfigJson for a Migration record matching the given filename. /// diff --git a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqliteMigrationRunMetaTests.cs b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqliteMigrationRunMetaTests.cs index 8d59716..454910b 100644 --- a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqliteMigrationRunMetaTests.cs +++ b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqliteMigrationRunMetaTests.cs @@ -91,6 +91,26 @@ public async Task ConnectionStrings_ShouldBeMasked() } } + [Fact] + public async Task FromReleaseVersion_IsTheHighestMigratedReleaseWhenTheRunStarted() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + + // The run row stamps ToReleaseVersion from the context, so the host is rebuilt for each target release. + await ctx.RebuildForAsync(MigrationCommand.MigrateUp, MigrationRunMode.Migrate, "Release_2.0"); + await ctx.MigrateUpAsync("Release_2.0"); + ctx.AssertSuccess(true); + await ctx.RebuildForAsync(MigrationCommand.MigrateUp, MigrationRunMode.Migrate); + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + ctx.AssertRunCount(2); + ctx.GetMigrationRun(1)!.FromReleaseVersion.Should().BeNull("nothing was migrated before the first run"); + ctx.AssertMigrationRun(1, new MigrationRunExpectation { ToReleaseVersion = "Release_2.0" }); + ctx.AssertMigrationRun(2, new MigrationRunExpectation { FromReleaseVersion = "Release_2.0" }); + } + [Fact] public async Task MetaCount_MatchesRunCount() { diff --git a/Raycoon.RayMigrator.Tests.Unit/P1_HighestMigratedReleaseTests.cs b/Raycoon.RayMigrator.Tests.Unit/P1_HighestMigratedReleaseTests.cs new file mode 100644 index 0000000..0dd9ed3 --- /dev/null +++ b/Raycoon.RayMigrator.Tests.Unit/P1_HighestMigratedReleaseTests.cs @@ -0,0 +1,49 @@ +using AwesomeAssertions; +using Raycoon.RayMigrator.Core.Configuration.Enums; +using Raycoon.RayMigrator.Core.Models; +using Raycoon.RayMigrator.Services; + +namespace Raycoon.RayMigrator.Tests.Unit; + +/// +/// P1: is the one definition of the "current release" +/// of a product and environment: Info reports it and every run stamps it into MigrationRun.FromReleaseVersion. +/// +public class HighestMigratedReleaseTests +{ + private static MigrationRecord Record(string release, MigrationStatus status) + => new() { ReleaseVersion = release, MigrationStatusId = status, Filename = "001.sql" }; + + [Fact] + public void GetHighestMigratedRelease_IgnoresRecordsThatAreNotMigrated() + { + var records = new[] + { + Record("Release_1.0", MigrationStatus.Migrated), + Record("Release_2.0", MigrationStatus.Migrated), + Record("Release_3.0", MigrationStatus.Failed), + Record("Release_4.0", MigrationStatus.NotMigrated), + }; + + MigrationService.GetHighestMigratedRelease(records).Should().Be("Release_2.0"); + } + + [Fact] + public void GetHighestMigratedRelease_ComparesOrdinalIgnoringCase() + { + var records = new[] + { + Record("release_1.0", MigrationStatus.Migrated), + Record("Release_1.1", MigrationStatus.Migrated), + }; + + MigrationService.GetHighestMigratedRelease(records).Should().Be("Release_1.1"); + } + + [Fact] + public void GetHighestMigratedRelease_NothingMigrated_ReturnsNull() + { + MigrationService.GetHighestMigratedRelease([]).Should().BeNull(); + MigrationService.GetHighestMigratedRelease([Record("Release_1.0", MigrationStatus.Failed)]).Should().BeNull(); + } +} diff --git a/Raycoon.RayMigrator.Tests.Unit/P1_TemplateExecutorEnvironmentIdTests.cs b/Raycoon.RayMigrator.Tests.Unit/P1_TemplateExecutorEnvironmentIdTests.cs index 7c4c64e..1899d73 100644 --- a/Raycoon.RayMigrator.Tests.Unit/P1_TemplateExecutorEnvironmentIdTests.cs +++ b/Raycoon.RayMigrator.Tests.Unit/P1_TemplateExecutorEnvironmentIdTests.cs @@ -38,7 +38,8 @@ private static (TemplateExecutor executor, IDal dal, Func get int productId = TestProductId, MigrationRunMode runMode = MigrationRunMode.Migrate, string scalarResult = "1,ok", - List>? readerRows = null) + List>? readerRows = null, + Action? configureContext = null) { var dal = Substitute.For(); DalParameterList? captured = null; @@ -58,6 +59,7 @@ private static (TemplateExecutor executor, IDal dal, Func get }); var ctx = BuildContext(environmentId, productId, runMode); + configureContext?.Invoke(ctx); var templateCache = BuildRealTemplateCache(); // TemplateExecutor has a public constructor @@ -212,6 +214,31 @@ public void RepositoryMigrationRunInsert_PassesEnvironmentIdInt() param.ParameterValue.Should().Be(TestEnvironmentId); } + [Fact] + public void RepositoryMigrationRunInsert_PassesTheReleaseTheRunStartsFrom() + { + var (executor, _, getCapture) = CreateExecutor(configureContext: ctx => ctx.MigrationState.FromReleaseVersion = "Release_1.0"); + + InvokeAndCapture(executor, e => e.RepositoryMigrationRunInsert("{}"), getCapture); + + var captured = getCapture()!; + captured.TryGetValue("FromReleaseVersion", out var param).Should().BeTrue(); + param!.ParameterValue.Should().Be("Release_1.0", "the run row records the highest Migrated release when the run started"); + } + + [Fact] + public void RepositoryMigrationRunInsert_FirstRun_PassesNullFromReleaseVersion() + { + var (executor, _, getCapture) = CreateExecutor(); + + InvokeAndCapture(executor, e => e.RepositoryMigrationRunInsert("{}"), getCapture); + + var captured = getCapture()!; + captured.TryGetValue("FromReleaseVersion", out var param).Should().BeTrue(); + param!.ParameterValue.Should().BeNull("nothing was migrated before a first run; the DAL binds null as DB NULL"); + param.ParameterType.Should().Be(typeof(string)); + } + [Fact] public void RepositoryMigrationRunInsert_DoesNotPassTextEnvironmentParameter() { From 86b90002cff4fa9506c1eebb4e83da8b80b5df09 Mon Sep 17 00:00:00 2001 From: danielpavic Date: Mon, 12 Oct 2026 22:07:26 +0200 Subject: [PATCH 04/14] Name the repository run and the target in the service results MigrationOperationResult, BaselineResult and MigrationRunInfo carry MigrationRunId, the id of the run row (0 when the run mode writes no row). The random RunId Guid was never stored anywhere, and the InitiatedBy placeholder was never filled; both are gone. MigrationFileResult carries TargetAlias and MigrationRecordId wherever an entry belongs to one target: TargetByTarget order, failures and rollbacks. A file that succeeded in FileByFile order keeps its one entry per file. MigrationRunInfo carries Environment and FromRelease; MigrationHistory, MigrationStatusInfo and BaselineResult carry Environment. GetHistoryAsync and GetStatusAsync read the runs of the requested environment only. Repository_MigrationRun_Select filtered by product alone, so the history listed the runs of every environment of the product with zero counts, and the status reported the last run of whichever environment ran last. The template takes @EnvironmentId in all five engines. MigrateUpAsync, MigrateDownAsync and BaselineAsync refuse a request whose TargetReleaseVersion differs from the MigrationContext, like the run-mode check does: the run row stamps the target release from the context, so a host that let them diverge recorded a release the run never aimed at. The engine test scenario rebuilds its host when the target release changes. Breaking change for consumers of the result DTOs (allowed before 1.0). --- CHANGELOG.md | 20 ++++ .../Repository_MigrationRun_Select.sql | 4 +- .../Repository_MigrationRun_Select.sql | 4 +- .../Repository_MigrationRun_Select.sql | 4 +- .../Repository_MigrationRun_Select.sql | 4 +- .../Repository_MigrationRun_Select.sql | 4 +- .../TemplateExecutor.cs | 5 +- .../Models/Results.cs | 30 +++++- .../MigrationService.cs | 91 +++++++++++++++---- .../Infrastructure/ScenarioContext.cs | 9 +- .../Tests/Features/SqliteInfoTests.cs | 34 +++++++ .../P3_BaselineAndInfoModelTests.cs | 5 +- 12 files changed, 180 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32c8019..ff902b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,20 @@ RayMigrator follows Semantic Versioning where applicable. API accessor isolates per async flow through its `AsyncLocal` anyway. `ProductAliasGuard` explains an unknown product alias with the same message the CLI showed, now also for hosts. +- The result contracts of `IMigrationService` name the repository run and the + target. `MigrationOperationResult`, `BaselineResult` and `MigrationRunInfo` + carry `MigrationRunId`, the id of the run row (0 when the run mode writes no + row); the random `RunId` Guid that was never stored is gone, as is the + `InitiatedBy` placeholder that nothing ever filled. `MigrationFileResult` + carries `TargetAlias` and `MigrationRecordId` wherever the entry belongs to + one target (TargetByTarget order, failures, rollbacks). `MigrationRunInfo` + carries `Environment` and `FromRelease`; `MigrationHistory`, + `MigrationStatusInfo` and `BaselineResult` carry `Environment`. +- `MigrateUpAsync`, `MigrateDownAsync` and `BaselineAsync` refuse a request + whose `TargetReleaseVersion` differs from the `MigrationContext`, like the + run-mode check does: the run row stamps the target release from the + context, so a host that let them diverge recorded a release the run never + aimed at. - `MigrationService` reads the configuration of the current run from its `MigrationContext` instead of from `IOptions`, and `TemplateCache` no longer takes the options: a host builds the cache once @@ -44,6 +58,12 @@ RayMigrator follows Semantic Versioning where applicable. ### Fixed +- `GetHistoryAsync` and `GetStatusAsync` read the runs of the requested + environment only. `Repository_MigrationRun_Select` filtered by product + alone, so the history listed the runs of every environment of the product + (with zero counts, because the record history is per environment) and the + status reported the last run of whichever environment ran last. The + template takes `@EnvironmentId` in all five engines. - `MigrationRun.FromReleaseVersion` holds the release the run started from: the highest Migrated release of the product and environment, read right before the run row is inserted, or `NULL` for a first run. Every run row diff --git a/Raycoon.RayMigrator.Database.MariaDb/Templates/Repository_MigrationRun_Select.sql b/Raycoon.RayMigrator.Database.MariaDb/Templates/Repository_MigrationRun_Select.sql index dac075a..cd82612 100644 --- a/Raycoon.RayMigrator.Database.MariaDb/Templates/Repository_MigrationRun_Select.sql +++ b/Raycoon.RayMigrator.Database.MariaDb/Templates/Repository_MigrationRun_Select.sql @@ -6,7 +6,7 @@ RayMigrator SQL Template TemplateType = "Repository_MigrationRun_Select" DatabaseType = "MariaDb" Author = "RAYCOON.com GmbH (https://raycoon.com)" -Version = "2026-04-18.1" +Version = "2026-10-12.1" [Description] Function = """ @@ -28,6 +28,7 @@ TableBaseName = "Table name prefix from Repository configuration" [Parameters] ProductId = "INT | REQUIRED | The product ID to query MigrationRuns for" +EnvironmentId = "INT | REQUIRED | The environment ID to query MigrationRuns for" Limit = "INT | REQUIRED | Maximum number of records to return" [ReturnValues] @@ -58,6 +59,7 @@ FROM {CFG:TableBaseName}migration_run WHERE product_id = @ProductId + AND environment_id = @EnvironmentId ORDER BY started_at DESC, id DESC LIMIT @Limit; diff --git a/Raycoon.RayMigrator.Database.MySql/Templates/Repository_MigrationRun_Select.sql b/Raycoon.RayMigrator.Database.MySql/Templates/Repository_MigrationRun_Select.sql index f7ff556..58d1c1e 100644 --- a/Raycoon.RayMigrator.Database.MySql/Templates/Repository_MigrationRun_Select.sql +++ b/Raycoon.RayMigrator.Database.MySql/Templates/Repository_MigrationRun_Select.sql @@ -6,7 +6,7 @@ RayMigrator SQL Template TemplateType = "Repository_MigrationRun_Select" DatabaseType = "MySql" Author = "RAYCOON.com GmbH (https://raycoon.com)" -Version = "2026-04-18.1" +Version = "2026-10-12.1" [Description] Function = """ @@ -28,6 +28,7 @@ TableBaseName = "Table name prefix from Repository configuration" [Parameters] ProductId = "INT | REQUIRED | The product ID to query MigrationRuns for" +EnvironmentId = "INT | REQUIRED | The environment ID to query MigrationRuns for" Limit = "INT | REQUIRED | Maximum number of records to return" [ReturnValues] @@ -58,6 +59,7 @@ FROM {CFG:TableBaseName}migration_run WHERE product_id = @ProductId + AND environment_id = @EnvironmentId ORDER BY started_at DESC, id DESC LIMIT @Limit; diff --git a/Raycoon.RayMigrator.Database.PostgreSQL/Templates/Repository_MigrationRun_Select.sql b/Raycoon.RayMigrator.Database.PostgreSQL/Templates/Repository_MigrationRun_Select.sql index be73351..b47524f 100644 --- a/Raycoon.RayMigrator.Database.PostgreSQL/Templates/Repository_MigrationRun_Select.sql +++ b/Raycoon.RayMigrator.Database.PostgreSQL/Templates/Repository_MigrationRun_Select.sql @@ -6,7 +6,7 @@ RayMigrator SQL Template TemplateType = "Repository_MigrationRun_Select" DatabaseType = "PostgreSQL" Author = "RAYCOON.com GmbH (https://raycoon.com)" -Version = "2026-04-18.1" +Version = "2026-10-12.1" [Description] Function = """ @@ -28,6 +28,7 @@ TableBaseName = "Table name prefix from Repository configuration (e.g., '' or 'R [Parameters] ProductId = "INT | REQUIRED | The product ID to query MigrationRuns for" +EnvironmentId = "INT | REQUIRED | The environment ID to query MigrationRuns for" Limit = "INT | REQUIRED | Maximum number of records to return" [ReturnValues] @@ -59,6 +60,7 @@ FROM {CFG:SchemaName}.{CFG:TableBaseName}migration_run WHERE product_id = @ProductId + AND environment_id = @EnvironmentId ORDER BY started_at DESC, id DESC LIMIT @Limit; diff --git a/Raycoon.RayMigrator.Database.SqlServer/Templates/Repository_MigrationRun_Select.sql b/Raycoon.RayMigrator.Database.SqlServer/Templates/Repository_MigrationRun_Select.sql index 631040b..47e4187 100644 --- a/Raycoon.RayMigrator.Database.SqlServer/Templates/Repository_MigrationRun_Select.sql +++ b/Raycoon.RayMigrator.Database.SqlServer/Templates/Repository_MigrationRun_Select.sql @@ -8,7 +8,7 @@ RayMigrator SQL Template TemplateType = "Repository_MigrationRun_Select" DatabaseType = "SqlServer" Author = "RAYCOON.com GmbH (https://raycoon.com)" -Version = "2026-04-18.1" +Version = "2026-10-12.1" [Description] Function = """ @@ -32,6 +32,7 @@ TableBaseName = "Table name prefix from Repository configuration (e.g., '' or 'R [Parameters] # SQL parameters bound at runtime ProductId = "INT | REQUIRED | The product ID to query MigrationRuns for" +EnvironmentId = "INT | REQUIRED | The environment ID to query MigrationRuns for" Limit = "INT | REQUIRED | Maximum number of records to return" [ReturnValues] @@ -67,6 +68,7 @@ BEGIN TRY [{CFG:SchemaName}].[{CFG:TableBaseName}MigrationRun] WHERE [ProductId] = @ProductId + AND [EnvironmentId] = @EnvironmentId ORDER BY [StartedAt] DESC, [Id] DESC; diff --git a/Raycoon.RayMigrator.Database.Sqlite/Templates/Repository_MigrationRun_Select.sql b/Raycoon.RayMigrator.Database.Sqlite/Templates/Repository_MigrationRun_Select.sql index 75c083a..817af3b 100644 --- a/Raycoon.RayMigrator.Database.Sqlite/Templates/Repository_MigrationRun_Select.sql +++ b/Raycoon.RayMigrator.Database.Sqlite/Templates/Repository_MigrationRun_Select.sql @@ -6,7 +6,7 @@ RayMigrator SQL Template TemplateType = "Repository_MigrationRun_Select" DatabaseType = "Sqlite" Author = "RAYCOON.com GmbH (https://raycoon.com)" -Version = "2026-04-18.1" +Version = "2026-10-12.1" [Description] Function = """ @@ -28,6 +28,7 @@ TableBaseName = "Table name prefix from Repository configuration" [Parameters] ProductId = "INTEGER | REQUIRED | The product ID to query MigrationRuns for" +EnvironmentId = "INTEGER | REQUIRED | The environment ID to query MigrationRuns for" Limit = "INTEGER | REQUIRED | Maximum number of records to return" [ReturnValues] @@ -58,6 +59,7 @@ FROM "{CFG:TableBaseName}MigrationRun" WHERE "ProductId" = @ProductId + AND "EnvironmentId" = @EnvironmentId ORDER BY "StartedAt" DESC, "Id" DESC LIMIT @Limit; diff --git a/Raycoon.RayMigrator.Infrastructure/TemplateExecutor.cs b/Raycoon.RayMigrator.Infrastructure/TemplateExecutor.cs index 0934f33..e5dcd5c 100644 --- a/Raycoon.RayMigrator.Infrastructure/TemplateExecutor.cs +++ b/Raycoon.RayMigrator.Infrastructure/TemplateExecutor.cs @@ -774,11 +774,12 @@ public void RepositoryMigrationUpdateHash( var templateType = TemplateType.Repository_MigrationRun_Select; var eventId = MigrationEvent.TemplateExecutionRepositoryMigrationRunSelect; - _logger.LogDebug(eventId, "Selecting MigrationRun records for product {ProductId} with limit {Limit}{MigrationContext}", - _ctxAccessor.Current.MigrationState.ProductId, limit, _ctxAccessor.Current.Clone); + _logger.LogDebug(eventId, "Selecting MigrationRun records for product {ProductId} and environment {EnvironmentId} with limit {Limit}{MigrationContext}", + _ctxAccessor.Current.MigrationState.ProductId, _ctxAccessor.Current.MigrationState.EnvironmentId, limit, _ctxAccessor.Current.Clone); DalParameterList dalParameterList = new DalParameterList(); dalParameterList.AddParameter(new DalParameter("ProductId", _ctxAccessor.Current.MigrationState.ProductId, typeof(int))); + dalParameterList.AddParameter(new DalParameter("EnvironmentId", _ctxAccessor.Current.MigrationState.EnvironmentId, typeof(int))); dalParameterList.AddParameter(new DalParameter("Limit", limit, typeof(int))); var template = _templateCache.GetRepositoryTemplate(templateType, _repository); diff --git a/Raycoon.RayMigrator.Services.Abstractions/Models/Results.cs b/Raycoon.RayMigrator.Services.Abstractions/Models/Results.cs index e1f6152..634f8bd 100644 --- a/Raycoon.RayMigrator.Services.Abstractions/Models/Results.cs +++ b/Raycoon.RayMigrator.Services.Abstractions/Models/Results.cs @@ -26,7 +26,11 @@ public abstract class OperationResult /// public class MigrationOperationResult : OperationResult { - public Guid RunId { get; set; } + /// + /// The repository id of the run row (MigrationRun.Id), or 0 when the run mode writes no run row + /// (Simulate, Validate) or the run failed before the row was inserted. + /// + public int MigrationRunId { get; set; } public string ProductAlias { get; set; } = string.Empty; public string Environment { get; set; } = string.Empty; public MigrationOperation Operation { get; set; } @@ -46,6 +50,17 @@ public class MigrationFileResult public string FileName { get; set; } = string.Empty; public string ReleaseVersion { get; set; } = string.Empty; public string TargetGroup { get; set; } = string.Empty; + + /// + /// The target the entry belongs to. Null when the entry summarizes all targets of the group, which is how + /// FileByFile order reports a file that succeeded (one entry per file); TargetByTarget order, failures and + /// rollbacks report the target. + /// + public string? TargetAlias { get; set; } + + /// The repository record of the file on the target, or 0 when none exists or applies. + public int MigrationRecordId { get; set; } + public bool Success { get; set; } public string? ErrorMessage { get; set; } public DateTime ExecutedAt { get; set; } @@ -98,6 +113,10 @@ public class HashUpdateResult : OperationResult public class BaselineResult : OperationResult { public string ProductAlias { get; set; } = string.Empty; + public string Environment { get; set; } = string.Empty; + + /// The repository id of the run row (MigrationRun.Id), or 0 when the run failed before it was inserted. + public int MigrationRunId { get; set; } public string? TargetReleaseVersion { get; set; } public int BaselinedFiles { get; set; } } @@ -108,6 +127,7 @@ public class BaselineResult : OperationResult public class MigrationStatusInfo { public string ProductAlias { get; set; } = string.Empty; + public string Environment { get; set; } = string.Empty; public string CurrentRelease { get; set; } = string.Empty; public DateTime? LastMigrationDate { get; set; } public int TotalMigrationsExecuted { get; set; } @@ -135,6 +155,7 @@ public class TargetGroupStatus public class MigrationHistory { public string ProductAlias { get; set; } = string.Empty; + public string Environment { get; set; } = string.Empty; public List Runs { get; set; } = new List(); } @@ -143,17 +164,20 @@ public class MigrationHistory /// public class MigrationRunInfo { + /// The repository id of the run row (MigrationRun.Id). public int MigrationRunId { get; set; } - public Guid RunId { get; set; } + public string Environment { get; set; } = string.Empty; public DateTime StartedAt { get; set; } public DateTime? CompletedAt { get; set; } public MigrationOperation Operation { get; set; } public MigrationRunResult Result { get; set; } public MigrationRunMode RunMode { get; set; } - public string? InitiatedBy { get; set; } public int TotalMigrations { get; set; } public int SuccessfulMigrations { get; set; } public int FailedMigrations { get; set; } + + /// The highest Migrated release when the run started; null for a first run. + public string? FromRelease { get; set; } public string? ToRelease { get; set; } } diff --git a/Raycoon.RayMigrator.Services/MigrationService.cs b/Raycoon.RayMigrator.Services/MigrationService.cs index 5a2d3f0..b7dd4d2 100644 --- a/Raycoon.RayMigrator.Services/MigrationService.cs +++ b/Raycoon.RayMigrator.Services/MigrationService.cs @@ -50,7 +50,6 @@ public MigrationService( public async Task MigrateUpAsync(MigrateUpRequest request) { var startTime = DateTime.UtcNow; - var runId = Guid.NewGuid(); var migrationResults = new List(); int successfulMigrations = 0; int failedMigrations = 0; @@ -67,6 +66,7 @@ public async Task MigrateUpAsync(MigrateUpRequest requ $"Product mismatch: context has {_ctxAccessor.Current.RayMigratorConsoleOptions.Product} but request has {request.ProductAlias}"); } EnsureRequestRunModeMatchesContext(request.RunMode); + EnsureRequestTargetReleaseMatchesContext(request.TargetReleaseVersion); // --- Phase 1: Initialization --- _ctxAccessor.Current.MigrationState.MigrationRunResult = MigrationRunResult.Running; @@ -110,7 +110,7 @@ public async Task MigrateUpAsync(MigrateUpRequest requ return new MigrationOperationResult { Success = true, - RunId = runId, + MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, ProductAlias = request.ProductAlias, Environment = request.Environment, Operation = MigrationOperation.MigrateUp, @@ -201,7 +201,7 @@ public async Task MigrateUpAsync(MigrateUpRequest requ return new MigrationOperationResult { Success = true, - RunId = runId, + MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, ProductAlias = request.ProductAlias, Environment = request.Environment, Operation = MigrationOperation.MigrateUp, @@ -299,7 +299,7 @@ public async Task MigrateUpAsync(MigrateUpRequest requ return new MigrationOperationResult { Success = false, - RunId = runId, + MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, ProductAlias = request.ProductAlias, Environment = request.Environment, Operation = MigrationOperation.MigrateUp, @@ -336,7 +336,7 @@ public async Task MigrateUpAsync(MigrateUpRequest requ return new MigrationOperationResult { Success = false, - RunId = runId, + MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, ProductAlias = request.ProductAlias, Environment = request.Environment, Operation = MigrationOperation.MigrateUp, @@ -358,7 +358,7 @@ public async Task MigrateUpAsync(MigrateUpRequest requ return new MigrationOperationResult { Success = true, - RunId = runId, + MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, ProductAlias = request.ProductAlias, Environment = request.Environment, Operation = MigrationOperation.MigrateUp, @@ -397,7 +397,7 @@ public async Task MigrateUpAsync(MigrateUpRequest requ return new MigrationOperationResult { Success = false, - RunId = runId, + MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, ProductAlias = request.ProductAlias, Environment = request.Environment, Operation = MigrationOperation.MigrateUp, @@ -623,6 +623,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( FileName = file.Filename, ReleaseVersion = file.ReleaseVersion, TargetGroup = file.TargetGroupAlias, + TargetAlias = _ctxAccessor.Current.MigrationState.TargetAlias, + MigrationRecordId = _ctxAccessor.Current.MigrationState.MigrationRecordId, Success = false, ErrorMessage = ex.Message, ExecutedAt = fileStartTime, @@ -643,6 +645,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( FileName = file.Filename, ReleaseVersion = file.ReleaseVersion, TargetGroup = file.TargetGroupAlias, + TargetAlias = _ctxAccessor.Current.MigrationState.TargetAlias, + MigrationRecordId = _ctxAccessor.Current.MigrationState.MigrationRecordId, Success = false, ErrorMessage = ex.Message, ExecutedAt = fileStartTime, @@ -706,6 +710,8 @@ internal async Task ExecuteTargetGroupTargetByTarget FileName = file.Filename, ReleaseVersion = file.ReleaseVersion, TargetGroup = file.TargetGroupAlias, + TargetAlias = targetOptions.Alias, + MigrationRecordId = finalizedId, Success = true, ExecutedAt = fileStartTime, Duration = DateTime.UtcNow - fileStartTime @@ -796,6 +802,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( FileName = file.Filename, ReleaseVersion = file.ReleaseVersion, TargetGroup = file.TargetGroupAlias, + TargetAlias = targetOptions.Alias, + MigrationRecordId = migrationRecordId, Success = false, ErrorMessage = $"MigrationErrorAction=Ignore: One or more SQL blocks failed in {file.Filename}", ExecutedAt = fileStartTime, @@ -822,6 +830,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( FileName = file.Filename, ReleaseVersion = file.ReleaseVersion, TargetGroup = file.TargetGroupAlias, + TargetAlias = targetOptions.Alias, + MigrationRecordId = migrationRecordId, Success = true, ExecutedAt = fileStartTime, Duration = DateTime.UtcNow - fileStartTime @@ -862,6 +872,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( FileName = file.Filename, ReleaseVersion = file.ReleaseVersion, TargetGroup = file.TargetGroupAlias, + TargetAlias = targetOptions.Alias, + MigrationRecordId = _ctxAccessor.Current.MigrationState.MigrationRecordId, Success = false, ErrorMessage = ex.Message, ExecutedAt = fileStartTime, @@ -882,6 +894,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( FileName = file.Filename, ReleaseVersion = file.ReleaseVersion, TargetGroup = file.TargetGroupAlias, + TargetAlias = targetOptions.Alias, + MigrationRecordId = _ctxAccessor.Current.MigrationState.MigrationRecordId, Success = false, ErrorMessage = ex.Message, ExecutedAt = fileStartTime, @@ -920,6 +934,23 @@ private void EnsureRequestRunModeMatchesContext(MigrationRunMode requestRunMode) } } + /// + /// The run row stamps ToReleaseVersion from the , while the execution path + /// reads the target release from the request. The CLI fills both from the same option; a programmatic caller + /// could let them diverge and stamp a release the run never aimed at. + /// + /// The request target release differs from the context. + private void EnsureRequestTargetReleaseMatchesContext(string? requestTargetRelease) + { + var contextTargetRelease = _ctxAccessor.Current.RayMigratorConsoleOptions.TargetReleaseVersion; + if (!string.Equals(requestTargetRelease, contextTargetRelease, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"TargetReleaseVersion mismatch: context has [{contextTargetRelease ?? "null"}] but request has [{requestTargetRelease ?? "null"}]. " + + "Build the MigrationContext for the target release the request is executed with."); + } + } + /// /// Determines whether a migration file can use the atomic shared-connection path, /// where target SQL blocks and repository status updates execute in a single transaction. @@ -1092,7 +1123,6 @@ internal void LogAtomicSharedConnectionTargets( public async Task MigrateDownAsync(MigrateDownRequest request) { var startTime = DateTime.UtcNow; - var runId = Guid.NewGuid(); try { @@ -1106,6 +1136,7 @@ public async Task MigrateDownAsync(MigrateDownRequest $"Product mismatch: context has {_ctxAccessor.Current.RayMigratorConsoleOptions.Product} but request has {request.ProductAlias}"); } EnsureRequestRunModeMatchesContext(request.RunMode); + EnsureRequestTargetReleaseMatchesContext(request.TargetReleaseVersion); // --- Validate mode: validate rollback file existence and parseability --- if (!request.RunMode.ShouldReadRepository()) @@ -1149,7 +1180,7 @@ public async Task MigrateDownAsync(MigrateDownRequest return new MigrationOperationResult { Success = warnings.Count == 0, - RunId = runId, + MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, ProductAlias = request.ProductAlias, Environment = request.Environment, Operation = MigrationOperation.MigrateDown, @@ -1210,7 +1241,7 @@ public async Task MigrateDownAsync(MigrateDownRequest return new MigrationOperationResult { Success = true, - RunId = runId, + MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, ProductAlias = request.ProductAlias, Environment = request.Environment, Operation = MigrationOperation.MigrateDown, @@ -1246,7 +1277,7 @@ public async Task MigrateDownAsync(MigrateDownRequest return new MigrationOperationResult { Success = true, - RunId = runId, + MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, ProductAlias = request.ProductAlias, Environment = request.Environment, Operation = MigrationOperation.MigrateDown, @@ -1289,7 +1320,7 @@ public async Task MigrateDownAsync(MigrateDownRequest return new MigrationOperationResult { Success = rollbackResult.AllSuccessful, - RunId = runId, + MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, ProductAlias = request.ProductAlias, Environment = request.Environment, Operation = MigrationOperation.MigrateDown, @@ -1327,7 +1358,7 @@ public async Task MigrateDownAsync(MigrateDownRequest return new MigrationOperationResult { Success = false, - RunId = runId, + MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, ProductAlias = request.ProductAlias, Environment = request.Environment, Operation = MigrationOperation.MigrateDown, @@ -1363,6 +1394,8 @@ public async Task BaselineAsync(BaselineRequest request) $"Product mismatch: context has {_ctxAccessor.Current.RayMigratorConsoleOptions.Product} but request has {request.ProductAlias}"); } + EnsureRequestTargetReleaseMatchesContext(request.TargetReleaseVersion); + // --- Phase 1: Initialization --- await InitializeRepositoryAsync(); @@ -1389,6 +1422,8 @@ public async Task BaselineAsync(BaselineRequest request) { Success = true, ProductAlias = request.ProductAlias, + Environment = request.Environment, + MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, TargetReleaseVersion = request.TargetReleaseVersion, BaselinedFiles = 0, Duration = DateTime.UtcNow - startTime, @@ -1410,6 +1445,8 @@ public async Task BaselineAsync(BaselineRequest request) { Success = true, ProductAlias = request.ProductAlias, + Environment = request.Environment, + MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, TargetReleaseVersion = request.TargetReleaseVersion, BaselinedFiles = 0, Duration = DateTime.UtcNow - startTime, @@ -1429,6 +1466,8 @@ public async Task BaselineAsync(BaselineRequest request) { Success = true, ProductAlias = request.ProductAlias, + Environment = request.Environment, + MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, TargetReleaseVersion = request.TargetReleaseVersion, BaselinedFiles = 0, Duration = DateTime.UtcNow - startTime, @@ -1553,6 +1592,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( { Success = true, ProductAlias = request.ProductAlias, + Environment = request.Environment, + MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, TargetReleaseVersion = request.TargetReleaseVersion, BaselinedFiles = baselinedCount, Duration = DateTime.UtcNow - startTime, @@ -1581,6 +1622,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( { Success = false, ProductAlias = request.ProductAlias, + Environment = request.Environment, + MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, TargetReleaseVersion = request.TargetReleaseVersion, Duration = DateTime.UtcNow - startTime, ErrorMessage = ex.Message, @@ -1657,7 +1700,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( record.Id, MigrationStatus.Failed, record.FileUpBlocksMigrated)); } - result.AddFailure(record.Filename, $"Required rollback file not found: {rollbackFilename}"); + result.AddFailure(record.Filename, $"Required rollback file not found: {rollbackFilename}", record.TargetAlias, record.Id); result.ErrorMessage = $"Rollback chain aborted: required rollback file missing for {record.Filename}"; return result; } @@ -1827,7 +1870,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdateRollback( rollbackFileInfo.FileUpConfigJson)); } - result.AddFailure(record.Filename, $"Atomic rollback failed: {blockEx.Message}"); + result.AddFailure(record.Filename, $"Atomic rollback failed: {blockEx.Message}", record.TargetAlias, record.Id); result.ErrorMessage = $"Rollback chain aborted: atomic rollback of {record.Filename} failed"; return result; @@ -1882,7 +1925,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdateRollback( rollbackFileInfo.FileUpBlocksTotal, rollbackFileInfo.FileUpConfigJson)); - result.AddFailure(record.Filename, $"Rollback block {blockIndex + 1} failed: {blockEx.Message}"); + result.AddFailure(record.Filename, $"Rollback block {blockIndex + 1} failed: {blockEx.Message}", record.TargetAlias, record.Id); result.ErrorMessage = $"Rollback chain aborted: block {blockIndex + 1} of {record.Filename} failed"; // Abort the entire rollback chain @@ -1949,6 +1992,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdateRollback( FileName = record.Filename, ReleaseVersion = record.ReleaseVersion, TargetGroup = record.TargetGroupAlias, + TargetAlias = record.TargetAlias, + MigrationRecordId = record.Id, Success = !fileHadBlockError, ExecutedAt = fileStartTime, Duration = DateTime.UtcNow - fileStartTime @@ -1965,7 +2010,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdateRollback( "CRITICAL: Rollback failed for migration {MigrationRecordId} ({Filename}). Rollback chain ABORTED.", record.Id, record.Filename); - result.AddFailure(record.Filename, ex.Message); + result.AddFailure(record.Filename, ex.Message, record.TargetAlias, record.Id); result.ErrorMessage = $"Rollback chain aborted at {record.Filename}: {ex.Message}"; // Abort the entire rollback chain @@ -4974,6 +5019,7 @@ public async Task GetStatusAsync(string productAlias) return new MigrationStatusInfo { ProductAlias = productAlias, + Environment = _ctxAccessor.Current.RayMigratorConsoleOptions.Environment, CurrentRelease = currentRelease, LastMigrationDate = lastMigrationDate, TotalMigrationsExecuted = migratedRecords.Count, @@ -5025,7 +5071,8 @@ public async Task GetHistoryAsync(string productAlias, int lim DateTime? finishedAt = row["FinishedAt"] != null && row["FinishedAt"] != DBNull.Value ? Convert.ToDateTime(row["FinishedAt"]) : null; - string? toRelease = row["ToReleaseVersion"]?.ToString(); + string? fromRelease = row["FromReleaseVersion"] as string; + string? toRelease = row["ToReleaseVersion"] as string; var runHistory = historyByRun.TryGetValue(runId, out var h) ? h : Array.Empty(); var (totalMigrations, successfulMigrations, failedMigrations, operation) = SummarizeRun(runHistory); @@ -5033,7 +5080,8 @@ public async Task GetHistoryAsync(string productAlias, int lim runs.Add(new MigrationRunInfo { MigrationRunId = runId, - RunId = Guid.Empty, // MigrationRun uses INT Id, not Guid + Environment = _ctxAccessor.Current.RayMigratorConsoleOptions.Environment, + FromRelease = fromRelease, StartedAt = startedAt, CompletedAt = finishedAt, Operation = operation, @@ -5052,6 +5100,7 @@ public async Task GetHistoryAsync(string productAlias, int lim return new MigrationHistory { ProductAlias = productAlias, + Environment = _ctxAccessor.Current.RayMigratorConsoleOptions.Environment, Runs = runs }; } @@ -5283,13 +5332,15 @@ public void AddWarning(string filename, string message) Messages.Add($"WARNING [{filename}]: {message}"); } - public void AddFailure(string filename, string message) + public void AddFailure(string filename, string message, string? targetAlias = null, int migrationRecordId = 0) { FailCount++; Messages.Add($"ERROR [{filename}]: {message}"); FileResults.Add(new MigrationFileResult { FileName = filename, + TargetAlias = targetAlias, + MigrationRecordId = migrationRecordId, Success = false, ErrorMessage = message, ExecutedAt = DateTime.UtcNow diff --git a/Raycoon.RayMigrator.Tests.Engine/Infrastructure/ScenarioContext.cs b/Raycoon.RayMigrator.Tests.Engine/Infrastructure/ScenarioContext.cs index a45bb0f..d00c480 100644 --- a/Raycoon.RayMigrator.Tests.Engine/Infrastructure/ScenarioContext.cs +++ b/Raycoon.RayMigrator.Tests.Engine/Infrastructure/ScenarioContext.cs @@ -123,6 +123,8 @@ public async Task MigrateDownAsync( public async Task BaselineAsync(string? toRelease = null, string[]? targetGroupAliases = null, string[]? targetGroupMigrationOrder = null, bool revealSensitiveData = false) { + await AlignHostRunModeAsync(MigrationCommand.Baseline, null, toRelease); + using var maskScope = SensitiveDataMasker.BeginScope(revealSensitiveData); SensitiveDataMasker.RegisterSensitiveData(_host.RayMigratorOptions); @@ -266,9 +268,12 @@ private async Task AlignHostRunModeAsync(MigrationCommand comm var hostOptions = HostConsoleOptions; var effective = runMode ?? hostOptions.RunMode; - if (effective != hostOptions.RunMode || hostOptions.Command != command) + // The context also stamps the target release into the run row and the engine refuses a request whose + // target release differs from it, so a different release rebuilds the host too. + if (effective != hostOptions.RunMode || hostOptions.Command != command + || !string.Equals(toRelease, hostOptions.TargetReleaseVersion, StringComparison.Ordinal)) { - await RebuildForAsync(command, effective, toRelease ?? hostOptions.TargetReleaseVersion, _environment); + await RebuildForAsync(command, effective, toRelease, _environment); } return effective; diff --git a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqliteInfoTests.cs b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqliteInfoTests.cs index 726ee3e..c840a5e 100644 --- a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqliteInfoTests.cs +++ b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqliteInfoTests.cs @@ -161,6 +161,40 @@ public async Task GetHistory_AfterMultipleRuns_ReturnsAllRuns() result.Runs.Count.Should().Be(2); } + /// + /// History and status are per environment: a run of another environment of the same product is not listed. + /// + [Fact] + public async Task GetHistory_ListsOnlyTheRunsOfTheRequestedEnvironment() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + + await using var ctx = await CreateScenario().BuildAsync(); + + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + // A second environment on the same repository: baseline writes a run row without touching the targets. + await ctx.RebuildForAsync(MigrationCommand.Baseline, MigrationRunMode.Migrate, environment: "Staging"); + await ctx.BaselineAsync(); + ctx.AssertSuccess(true); + ctx.AssertRunCount(2); + + await ctx.RebuildForAsync(MigrationCommand.Info, MigrationRunMode.Migrate, environment: "Staging"); + var staging = await ctx.GetHistoryAsync(); + staging.Environment.Should().Be("Staging"); + staging.Runs.Should().ContainSingle("the up run belongs to the Docker environment") + .Which.Operation.Should().Be(MigrationOperation.Baseline); + + await ctx.RebuildForAsync(MigrationCommand.Info, MigrationRunMode.Migrate); + var docker = await ctx.GetHistoryAsync(); + docker.Environment.Should().Be("Docker"); + docker.Runs.Should().ContainSingle("the baseline run belongs to the Staging environment") + .Which.Operation.Should().Be(MigrationOperation.MigrateUp); + docker.Runs[0].FromRelease.Should().BeNull("nothing was migrated before the first run"); + docker.Runs[0].MigrationRunId.Should().BePositive(); + } + /// /// I8: GetHistory should return run details including result and migration count. /// diff --git a/Raycoon.RayMigrator.Tests.Unit/P3_BaselineAndInfoModelTests.cs b/Raycoon.RayMigrator.Tests.Unit/P3_BaselineAndInfoModelTests.cs index 066d96e..f0d5b69 100644 --- a/Raycoon.RayMigrator.Tests.Unit/P3_BaselineAndInfoModelTests.cs +++ b/Raycoon.RayMigrator.Tests.Unit/P3_BaselineAndInfoModelTests.cs @@ -167,11 +167,12 @@ public void MigrationRunInfo_DefaultValues_AreCorrect() { var run = new MigrationRunInfo(); - run.RunId.Should().Be(Guid.Empty); + run.MigrationRunId.Should().Be(0); + run.Environment.Should().BeEmpty(); run.TotalMigrations.Should().Be(0); run.SuccessfulMigrations.Should().Be(0); run.FailedMigrations.Should().Be(0); - run.InitiatedBy.Should().BeNull(); + run.FromRelease.Should().BeNull(); run.CompletedAt.Should().BeNull(); run.ToRelease.Should().BeNull(); } From 9870f62bf911b1100e6fc2352fb5e673d6c11b19 Mon Sep 17 00:00:00 2001 From: danielpavic Date: Mon, 12 Oct 2026 22:12:01 +0200 Subject: [PATCH 05/14] Raise typed progress events for hosts A host that shows the progress of a run had to parse log text: the only signal per file was the "Migration successful | ..." message, which has no event id and changes with the wording. MigrationContext.ProgressObserver now takes an IMigrationProgressObserver that receives RunStartedEvent after the run row exists (with its id), FileStartedEvent and FileCompletedEvent for every file on every target with the record id and a FileOutcome (migrated, recovered, simulated, failed, failed-ignored, rolled back, rollback failed, baselined), BlockCompletedEvent for every executed SQL block on the block-by-block, atomic and simulate paths, and RunCompletedEvent with the result. Rollbacks report MigrateDown or Rollback (error recovery) as their operation. The public operations are now thin wrappers around the existing bodies: they reset the run-started flag and raise the completion from the result, so an operation that ends before its run row exists still reports a matching start. An observer that throws is logged and the run continues; the default observer ignores every event, and the CLI sets none. The engine test scenario keeps an observer across host rebuilds, and the SQLite collection asserts the sequences for migrate-up, simulate, migrate-down, baseline, a failing file and a throwing observer. --- CHANGELOG.md | 12 + Raycoon.RayMigrator.Core/MigrationContext.cs | 8 + .../Progress/BlockCompletedEvent.cs | 30 +++ .../Progress/FileCompletedEvent.cs | 26 +++ .../Progress/FileOutcome.cs | 33 +++ .../Progress/FileStartedEvent.cs | 28 +++ .../Progress/IMigrationProgressObserver.cs | 26 +++ .../Progress/NullMigrationProgressObserver.cs | 33 +++ .../Progress/RunCompletedEvent.cs | 30 +++ .../Progress/RunStartedEvent.cs | 24 ++ .../MigrationService.cs | 215 +++++++++++++++++- .../RecordingProgressObserver.cs | 23 ++ .../Infrastructure/ScenarioContext.cs | 16 ++ .../Features/SqliteProgressEventsTests.cs | 176 ++++++++++++++ 14 files changed, 678 insertions(+), 2 deletions(-) create mode 100644 Raycoon.RayMigrator.Core/Progress/BlockCompletedEvent.cs create mode 100644 Raycoon.RayMigrator.Core/Progress/FileCompletedEvent.cs create mode 100644 Raycoon.RayMigrator.Core/Progress/FileOutcome.cs create mode 100644 Raycoon.RayMigrator.Core/Progress/FileStartedEvent.cs create mode 100644 Raycoon.RayMigrator.Core/Progress/IMigrationProgressObserver.cs create mode 100644 Raycoon.RayMigrator.Core/Progress/NullMigrationProgressObserver.cs create mode 100644 Raycoon.RayMigrator.Core/Progress/RunCompletedEvent.cs create mode 100644 Raycoon.RayMigrator.Core/Progress/RunStartedEvent.cs create mode 100644 Raycoon.RayMigrator.Tests.Engine/Infrastructure/RecordingProgressObserver.cs create mode 100644 Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqliteProgressEventsTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index ff902b2..579c50a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ RayMigrator follows Semantic Versioning where applicable. ## [Unreleased] +### Added + +- Typed progress events for hosts. `MigrationContext.ProgressObserver` takes an + `IMigrationProgressObserver` (namespace `Raycoon.RayMigrator.Core.Progress`) + that receives `RunStartedEvent` (after the run row exists, with its id), + `FileStartedEvent` and `FileCompletedEvent` for every file on every target + (migration, rollback and baseline, with the record id and a `FileOutcome`), + `BlockCompletedEvent` for every executed SQL block, and `RunCompletedEvent` + with the result. A host shows progress without parsing log text. An + observer that throws is logged and the run continues. The default observer + ignores every event; the CLI does not use them. + ### Changed - `IMigrationContextFactory` is the entry point of a host for one run, the CLI diff --git a/Raycoon.RayMigrator.Core/MigrationContext.cs b/Raycoon.RayMigrator.Core/MigrationContext.cs index e63123e..8fd8e6c 100644 --- a/Raycoon.RayMigrator.Core/MigrationContext.cs +++ b/Raycoon.RayMigrator.Core/MigrationContext.cs @@ -2,6 +2,7 @@ using Raycoon.RayMigrator.Core.Configuration.Enums; using Raycoon.RayMigrator.Core.Configuration.Options; using Raycoon.RayMigrator.Core.Configuration.Validation; +using Raycoon.RayMigrator.Core.Progress; using Raycoon.RayMigrator.Database.Common; using Raycoon.RayMigrator.Shared.Exceptions; @@ -109,6 +110,13 @@ public override string ToString() /// public ConcurrentDictionary DalSpecificPropertiesDictionary { get; set; } = new(); + /// + /// Receives the lifecycle events of the run (run, files on targets, SQL blocks). A host sets it before it + /// calls the service; the default ignores every event. Not copied by , which exists for + /// logging only. + /// + public IMigrationProgressObserver ProgressObserver { get; set; } = NullMigrationProgressObserver.Instance; + /// /// Getting a deep copy regarding MigrationState for logging purposes. /// diff --git a/Raycoon.RayMigrator.Core/Progress/BlockCompletedEvent.cs b/Raycoon.RayMigrator.Core/Progress/BlockCompletedEvent.cs new file mode 100644 index 0000000..52a8de5 --- /dev/null +++ b/Raycoon.RayMigrator.Core/Progress/BlockCompletedEvent.cs @@ -0,0 +1,30 @@ +using Raycoon.RayMigrator.Core.Configuration.Enums; + +namespace Raycoon.RayMigrator.Core.Progress; + +/// +/// Raised after a SQL block of a file was executed on a target. On the atomic shared-connection path the blocks +/// of a file and the repository update commit together at the end of the file, so a block reported here is not yet +/// durable until follows. +/// +/// MigrateUp, MigrateDown or Rollback. +/// The release the file belongs to. +/// The target group. +/// The target. +/// The file name without path. +/// The repository record, or 0 when the run mode writes no records. +/// The 1-based number of the block. +/// The number of blocks of the file. +/// Whether the block ran without error. False only under the Ignore error action; otherwise a failing block ends the file. +/// The error of a failed block. +public sealed record BlockCompletedEvent( + MigrationOperation Operation, + string ReleaseVersion, + string TargetGroupAlias, + string TargetAlias, + string FileName, + int MigrationRecordId, + int BlockNumber, + int BlocksTotal, + bool Success, + string? ErrorMessage); diff --git a/Raycoon.RayMigrator.Core/Progress/FileCompletedEvent.cs b/Raycoon.RayMigrator.Core/Progress/FileCompletedEvent.cs new file mode 100644 index 0000000..6ddaea0 --- /dev/null +++ b/Raycoon.RayMigrator.Core/Progress/FileCompletedEvent.cs @@ -0,0 +1,26 @@ +using Raycoon.RayMigrator.Core.Configuration.Enums; + +namespace Raycoon.RayMigrator.Core.Progress; + +/// +/// Raised when a file finished on a target. +/// +/// MigrateUp, MigrateDown, Rollback (error recovery) or Baseline. +/// The release the file belongs to. +/// The target group. +/// The target. +/// The file name without path. +/// The repository record, or 0 when the run mode writes no records. +/// How the file ended. +/// How long the file took on the target. +/// The error of a failed file. +public sealed record FileCompletedEvent( + MigrationOperation Operation, + string ReleaseVersion, + string TargetGroupAlias, + string TargetAlias, + string FileName, + int MigrationRecordId, + FileOutcome Outcome, + TimeSpan Duration, + string? ErrorMessage); diff --git a/Raycoon.RayMigrator.Core/Progress/FileOutcome.cs b/Raycoon.RayMigrator.Core/Progress/FileOutcome.cs new file mode 100644 index 0000000..1f6494c --- /dev/null +++ b/Raycoon.RayMigrator.Core/Progress/FileOutcome.cs @@ -0,0 +1,33 @@ +namespace Raycoon.RayMigrator.Core.Progress; + +/// +/// How a file ended on a target, as reported by . +/// +public enum FileOutcome +{ + Undefined = 0, + + /// Every block ran and the record is Migrated. + Migrated = 10, + + /// A previous run had executed every block but not finalized the record; this run finalized it without executing SQL. + Recovered = 20, + + /// Simulate or Validate run mode: the blocks were walked but no SQL ran. + Simulated = 30, + + /// The file failed and the run handles the error (abort or rollback, depending on the error action). + Failed = 40, + + /// The file failed and the run continues because the error action is Ignore. + FailedIgnored = 50, + + /// The rollback file ran and the record is NotMigrated. + RolledBack = 60, + + /// The rollback file failed. + RollbackFailed = 70, + + /// Baseline recorded the file as Migrated without executing it. + Baselined = 80, +} diff --git a/Raycoon.RayMigrator.Core/Progress/FileStartedEvent.cs b/Raycoon.RayMigrator.Core/Progress/FileStartedEvent.cs new file mode 100644 index 0000000..b977953 --- /dev/null +++ b/Raycoon.RayMigrator.Core/Progress/FileStartedEvent.cs @@ -0,0 +1,28 @@ +using Raycoon.RayMigrator.Core.Configuration.Enums; + +namespace Raycoon.RayMigrator.Core.Progress; + +/// +/// Raised when a file is about to run on a target. +/// +/// MigrateUp, MigrateDown, Rollback (error recovery) or Baseline. +/// The release the file belongs to. +/// The target group. +/// The target the file runs on. +/// The file name without path. +/// The position of the file in the execution order. +/// The repository record of the file on the target, or 0 when the run mode writes no records. +/// The number of SQL blocks of the file; 1 for a file that runs through a CLI tool. +/// The number of leading blocks a previous run already committed and this run skips. +/// When the file started. +public sealed record FileStartedEvent( + MigrationOperation Operation, + string ReleaseVersion, + string TargetGroupAlias, + string TargetAlias, + string FileName, + int FileOrderId, + int MigrationRecordId, + int BlocksTotal, + int ResumeFromBlock, + DateTime StartedAtUtc); diff --git a/Raycoon.RayMigrator.Core/Progress/IMigrationProgressObserver.cs b/Raycoon.RayMigrator.Core/Progress/IMigrationProgressObserver.cs new file mode 100644 index 0000000..8f9899b --- /dev/null +++ b/Raycoon.RayMigrator.Core/Progress/IMigrationProgressObserver.cs @@ -0,0 +1,26 @@ +namespace Raycoon.RayMigrator.Core.Progress; + +/// +/// Receives the lifecycle events of one run: the run itself, every file on every target, and every SQL block. +/// A host sets its observer on before it calls the service; the +/// engine calls the methods synchronously on the thread that executes the run, so an implementation must be fast +/// and must not throw. Logging stays with Serilog; these events exist so that a host can show progress without +/// parsing log text. +/// +public interface IMigrationProgressObserver +{ + /// A migrate-up, migrate-down or baseline run has started. Raised after the run row exists. + void OnRunStarted(RunStartedEvent e); + + /// A file is about to run on a target (migration, rollback or baseline). + void OnFileStarted(FileStartedEvent e); + + /// A SQL block of a file was executed on a target. Not raised for files that run through a CLI tool. + void OnBlockCompleted(BlockCompletedEvent e); + + /// A file finished on a target, with its outcome. + void OnFileCompleted(FileCompletedEvent e); + + /// The run finished, with its result. Raised exactly once per . + void OnRunCompleted(RunCompletedEvent e); +} diff --git a/Raycoon.RayMigrator.Core/Progress/NullMigrationProgressObserver.cs b/Raycoon.RayMigrator.Core/Progress/NullMigrationProgressObserver.cs new file mode 100644 index 0000000..d369f7c --- /dev/null +++ b/Raycoon.RayMigrator.Core/Progress/NullMigrationProgressObserver.cs @@ -0,0 +1,33 @@ +namespace Raycoon.RayMigrator.Core.Progress; + +/// +/// The observer of a context whose host did not set one. Ignores every event. +/// +public sealed class NullMigrationProgressObserver : IMigrationProgressObserver +{ + public static NullMigrationProgressObserver Instance { get; } = new(); + + private NullMigrationProgressObserver() + { + } + + public void OnRunStarted(RunStartedEvent e) + { + } + + public void OnFileStarted(FileStartedEvent e) + { + } + + public void OnBlockCompleted(BlockCompletedEvent e) + { + } + + public void OnFileCompleted(FileCompletedEvent e) + { + } + + public void OnRunCompleted(RunCompletedEvent e) + { + } +} diff --git a/Raycoon.RayMigrator.Core/Progress/RunCompletedEvent.cs b/Raycoon.RayMigrator.Core/Progress/RunCompletedEvent.cs new file mode 100644 index 0000000..6d759e0 --- /dev/null +++ b/Raycoon.RayMigrator.Core/Progress/RunCompletedEvent.cs @@ -0,0 +1,30 @@ +using Raycoon.RayMigrator.Core.Configuration.Enums; + +namespace Raycoon.RayMigrator.Core.Progress; + +/// +/// Raised once when a migrate-up, migrate-down or baseline run finished. +/// +/// The product of the run. +/// The environment of the run. +/// The command that ran. +/// The repository id of the run row, or 0 when none was written. +/// Whether the operation succeeded. +/// The persisted run result for migrate-up and migrate-down; null for baseline. +/// The entries of the result: files, or file and target pairs, as the operation counts them. +/// The entries that succeeded. +/// The entries that failed. +/// How long the run took. +/// The error of a failed run. +public sealed record RunCompletedEvent( + string ProductAlias, + string Environment, + MigrationCommand Command, + int MigrationRunId, + bool Success, + MigrationRunResult? Result, + int TotalFiles, + int SucceededFiles, + int FailedFiles, + TimeSpan Duration, + string? ErrorMessage); diff --git a/Raycoon.RayMigrator.Core/Progress/RunStartedEvent.cs b/Raycoon.RayMigrator.Core/Progress/RunStartedEvent.cs new file mode 100644 index 0000000..84d485d --- /dev/null +++ b/Raycoon.RayMigrator.Core/Progress/RunStartedEvent.cs @@ -0,0 +1,24 @@ +using Raycoon.RayMigrator.Core.Configuration.Enums; + +namespace Raycoon.RayMigrator.Core.Progress; + +/// +/// Raised once per migrate-up, migrate-down or baseline run, after the run row exists. +/// +/// The product of the run. +/// The environment of the run. +/// The command that runs. +/// Migrate, Simulate or Validate. +/// The repository id of the run row, or 0 when the run mode writes no run row. +/// The highest Migrated release when the run started; null for a first run. +/// The release the run aims at, or null for "everything". +/// When the run started. +public sealed record RunStartedEvent( + string ProductAlias, + string Environment, + MigrationCommand Command, + MigrationRunMode RunMode, + int MigrationRunId, + string? FromRelease, + string? TargetRelease, + DateTime StartedAtUtc); diff --git a/Raycoon.RayMigrator.Services/MigrationService.cs b/Raycoon.RayMigrator.Services/MigrationService.cs index b7dd4d2..0519307 100644 --- a/Raycoon.RayMigrator.Services/MigrationService.cs +++ b/Raycoon.RayMigrator.Services/MigrationService.cs @@ -10,6 +10,7 @@ using Raycoon.RayMigrator.Core.Configuration.Replacer; using Raycoon.RayMigrator.Core.Extensions; using Raycoon.RayMigrator.Core.Models; +using Raycoon.RayMigrator.Core.Progress; using Raycoon.RayMigrator.Database; using Raycoon.RayMigrator.Database.Common; using Raycoon.RayMigrator.Infrastructure; @@ -31,6 +32,9 @@ public class MigrationService : IMigrationService private readonly IServiceProvider _serviceProvider; private readonly ICliToolExecutor _cliToolExecutor; + /// Whether fired for the current operation; the wrapper resets it. + private bool _runStartedReported; + public MigrationService( ILogger logger, TemplateExecutor templateExecutor, @@ -48,6 +52,28 @@ public MigrationService( #region MigrateUp public async Task MigrateUpAsync(MigrateUpRequest request) + { + _runStartedReported = false; + var runStart = DateTime.UtcNow; + MigrationOperationResult result; + try + { + result = await MigrateUpCoreAsync(request); + } + catch (Exception ex) + { + ReportRunCompleted(MigrationCommand.MigrateUp, request.RunMode, request.TargetReleaseVersion, + false, null, 0, 0, 0, DateTime.UtcNow - runStart, ex.Message); + throw; + } + + ReportRunCompleted(MigrationCommand.MigrateUp, request.RunMode, request.TargetReleaseVersion, + result.Success, result.Result, result.TotalMigrations, result.SuccessfulMigrations, result.FailedMigrations, + result.Duration, result.ErrorMessage); + return result; + } + + private async Task MigrateUpCoreAsync(MigrateUpRequest request) { var startTime = DateTime.UtcNow; var migrationResults = new List(); @@ -96,6 +122,8 @@ public async Task MigrateUpAsync(MigrateUpRequest requ await RepositoryMigrationRunInsertWithAutoFix(settingsJson); } + ReportRunStarted(MigrationCommand.MigrateUp, request.RunMode, request.TargetReleaseVersion); + // --- Phase 2: File Discovery & Preparation --- var productOptions = _ctxAccessor.Current.RayMigratorOptions.Products!.First(p => p.Alias == request.ProductAlias); var migrationFiles = DiscoverAndPrepareMigrationFiles(productOptions, request.Environment); @@ -460,6 +488,7 @@ internal async Task ExecuteTargetGroupFileByFile( if (finalizedId > 0) { successfullyMigratedRecords.Add((file, finalizedId, targetOptions.Alias!)); + ReportFileCompleted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, finalizedId, FileOutcome.Recovered, fileStartTime); _logger.LogInformation( "Migration recovered | Product: {Product} | Env: {Environment} | Release: {Release} | TargetGroup: {TargetGroup} | Target: {Target} | File: {Filename} | MigrationRecordId: {MigrationRecordId} (finalized from previous incomplete run)", productOptions.Alias, _ctxAccessor.Current.RayMigratorConsoleOptions.Environment, file.ReleaseVersion, targetGroupOptions.Alias, targetOptions.Alias, file.Filename, finalizedId); @@ -508,6 +537,8 @@ internal async Task ExecuteTargetGroupFileByFile( // Check if this file+target can resume from a previous partial execution int startFromBlock = FindResumableBlock(file, targetOptions.Alias!, existingRecords); + ReportFileStarted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, migrationRecordId, startFromBlock, fileStartTime); + // Branch: CLI tool or DAL execution int succeededBlocks, failedBlocks; bool atomicCommitCompleted; @@ -529,6 +560,8 @@ internal async Task ExecuteTargetGroupFileByFile( if (failedBlocks > 0) { + ReportFileCompleted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, migrationRecordId, FileOutcome.FailedIgnored, fileStartTime, + $"{failedBlocks}/{file.FileUpBlocksTotal} block(s) failed"); // Block-level failures with Ignore: mark as Failed, skip remaining targets for this file fileHadBlockFailures = true; @@ -555,6 +588,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( } successfullyMigratedRecords.Add((file, migrationRecordId, targetOptions.Alias!)); + ReportFileCompleted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, migrationRecordId, + request.RunMode.ShouldExecuteSql() ? FileOutcome.Migrated : FileOutcome.Simulated, fileStartTime); _logger.LogInformation( "Migration successful | Product: {Product} | Env: {Environment} | Release: {Release} | TargetGroup: {TargetGroup} | Target: {Target} | File: {Filename} | MigrationRecordId: {MigrationRecordId} | SqlBlocks: {SqlBlocksTotal}", @@ -590,6 +625,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( } catch (Exception ex) { + ReportFileCompleted(MigrationOperation.MigrateUp, file, _ctxAccessor.Current.MigrationState.TargetAlias, + _ctxAccessor.Current.MigrationState.MigrationRecordId, ignoreErrors ? FileOutcome.FailedIgnored : FileOutcome.Failed, fileStartTime, ex.Message); _logger.LogError(ex, "Migration FAILED | Product: {Product} | Env: {Environment} | Release: {Release} | TargetGroup: {TargetGroup} | Target: {Target} | File: {Filename} | MigrationRecordId: {MigrationRecordId} | SqlBlock: {SqlBlock}/{SqlBlocksTotal}", productOptions.Alias, _ctxAccessor.Current.RayMigratorConsoleOptions.Environment, file.ReleaseVersion, targetGroupOptions.Alias, _ctxAccessor.Current.MigrationState.TargetAlias, file.Filename, _ctxAccessor.Current.MigrationState.MigrationRecordId, _ctxAccessor.Current.MigrationState.FileBlockId, file.FileUpBlocksTotal); @@ -700,6 +737,7 @@ internal async Task ExecuteTargetGroupTargetByTarget if (finalizedId > 0) { successfullyMigratedRecords.Add((file, finalizedId, targetOptions.Alias!)); + ReportFileCompleted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, finalizedId, FileOutcome.Recovered, fileStartTime); _logger.LogInformation( "Migration recovered | Product: {Product} | Env: {Environment} | Release: {Release} | TargetGroup: {TargetGroup} | Target: {Target} | File: {Filename} | MigrationRecordId: {MigrationRecordId} (finalized from previous incomplete run)", productOptions.Alias, _ctxAccessor.Current.RayMigratorConsoleOptions.Environment, file.ReleaseVersion, targetGroupOptions.Alias, targetOptions.Alias, file.Filename, finalizedId); @@ -761,6 +799,8 @@ internal async Task ExecuteTargetGroupTargetByTarget // Check if this file+target can resume from a previous partial execution int startFromBlock = FindResumableBlock(file, targetOptions.Alias!, existingRecords); + ReportFileStarted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, migrationRecordId, startFromBlock, fileStartTime); + // Branch: CLI tool or DAL execution int succeededBlocks, failedBlocks; bool atomicCommitCompleted; @@ -782,6 +822,8 @@ internal async Task ExecuteTargetGroupTargetByTarget if (failedBlocks > 0) { + ReportFileCompleted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, migrationRecordId, FileOutcome.FailedIgnored, fileStartTime, + $"{failedBlocks}/{file.FileUpBlocksTotal} block(s) failed"); // Block-level failures with Ignore: mark as Failed, continue to next file _logger.LogWarning( "MigrationErrorAction=Ignore: {FailedBlocks}/{TotalBlocks} block(s) failed in {Filename} on target {Target}. Continuing with next file.", @@ -819,6 +861,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( } successfullyMigratedRecords.Add((file, migrationRecordId, targetOptions.Alias!)); + ReportFileCompleted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, migrationRecordId, + request.RunMode.ShouldExecuteSql() ? FileOutcome.Migrated : FileOutcome.Simulated, fileStartTime); _logger.LogInformation( "Migration successful | Product: {Product} | Env: {Environment} | Release: {Release} | TargetGroup: {TargetGroup} | Target: {Target} | File: {Filename} | MigrationRecordId: {MigrationRecordId} | SqlBlocks: {SqlBlocksTotal}", @@ -839,6 +883,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( } catch (Exception ex) { + ReportFileCompleted(MigrationOperation.MigrateUp, file, _ctxAccessor.Current.MigrationState.TargetAlias, + _ctxAccessor.Current.MigrationState.MigrationRecordId, ignoreErrors ? FileOutcome.FailedIgnored : FileOutcome.Failed, fileStartTime, ex.Message); _logger.LogError(ex, "Migration FAILED | Product: {Product} | Env: {Environment} | Release: {Release} | TargetGroup: {TargetGroup} | Target: {Target} | File: {Filename} | MigrationRecordId: {MigrationRecordId} | SqlBlock: {SqlBlock}/{SqlBlocksTotal}", productOptions.Alias, _ctxAccessor.Current.RayMigratorConsoleOptions.Environment, file.ReleaseVersion, targetGroupOptions.Alias, targetOptions.Alias, file.Filename, _ctxAccessor.Current.MigrationState.MigrationRecordId, _ctxAccessor.Current.MigrationState.FileBlockId, file.FileUpBlocksTotal); @@ -910,6 +956,87 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( return result; } + #region Progress events + + private static MigrationOperation RollbackOperation(bool isErrorRecovery) + => isErrorRecovery ? MigrationOperation.Rollback : MigrationOperation.MigrateDown; + + /// + /// Hands an event to the observer of the current context. An observer must not throw; one that does is logged + /// and the run continues, because a broken progress display must never abort a migration. + /// + private void Report(string eventName, Action report) + { + try + { + report(_ctxAccessor.Current.ProgressObserver); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "The progress observer threw on {Event}; the run continues", eventName); + } + } + + /// + /// Raised once per operation, after the run row exists so that its id is known. The wrapper of each operation + /// calls it again before , which covers an operation that ended before the row + /// was inserted (Validate mode, a failure during initialization). + /// + private void ReportRunStarted(MigrationCommand command, MigrationRunMode runMode, string? targetRelease) + { + if (_runStartedReported) + { + return; + } + + _runStartedReported = true; + var ctx = _ctxAccessor.Current; + Report(nameof(IMigrationProgressObserver.OnRunStarted), o => o.OnRunStarted(new RunStartedEvent( + ctx.RayMigratorConsoleOptions.Product, ctx.RayMigratorConsoleOptions.Environment, command, runMode, + ctx.MigrationState.MigrationRunId, ctx.MigrationState.FromReleaseVersion, targetRelease, DateTime.UtcNow))); + } + + private void ReportRunCompleted( + MigrationCommand command, MigrationRunMode runMode, string? targetRelease, + bool success, MigrationRunResult? result, int totalFiles, int succeededFiles, int failedFiles, + TimeSpan duration, string? errorMessage) + { + ReportRunStarted(command, runMode, targetRelease); + var ctx = _ctxAccessor.Current; + Report(nameof(IMigrationProgressObserver.OnRunCompleted), o => o.OnRunCompleted(new RunCompletedEvent( + ctx.RayMigratorConsoleOptions.Product, ctx.RayMigratorConsoleOptions.Environment, command, + ctx.MigrationState.MigrationRunId, success, result, totalFiles, succeededFiles, failedFiles, duration, errorMessage))); + } + + private void ReportFileStarted(MigrationOperation operation, MigrationFileInfo file, string targetAlias, int migrationRecordId, int resumeFromBlock, DateTime startedAtUtc) + => ReportFileStarted(operation, file.ReleaseVersion, file.TargetGroupAlias, targetAlias, file.Filename, file.FileOrderId, migrationRecordId, + Math.Max(file.FileUpBlocksTotal, 1), resumeFromBlock, startedAtUtc); + + private void ReportFileStarted( + MigrationOperation operation, string releaseVersion, string targetGroupAlias, string targetAlias, string fileName, + int fileOrderId, int migrationRecordId, int blocksTotal, int resumeFromBlock, DateTime startedAtUtc) + => Report(nameof(IMigrationProgressObserver.OnFileStarted), o => o.OnFileStarted(new FileStartedEvent( + operation, releaseVersion, targetGroupAlias, targetAlias, fileName, fileOrderId, migrationRecordId, blocksTotal, resumeFromBlock, startedAtUtc))); + + private void ReportFileCompleted(MigrationOperation operation, MigrationFileInfo file, string targetAlias, int migrationRecordId, FileOutcome outcome, DateTime startedAtUtc, string? errorMessage = null) + => ReportFileCompleted(operation, file.ReleaseVersion, file.TargetGroupAlias, targetAlias, file.Filename, migrationRecordId, outcome, startedAtUtc, errorMessage); + + private void ReportFileCompleted( + MigrationOperation operation, string releaseVersion, string targetGroupAlias, string targetAlias, string fileName, + int migrationRecordId, FileOutcome outcome, DateTime startedAtUtc, string? errorMessage) + => Report(nameof(IMigrationProgressObserver.OnFileCompleted), o => o.OnFileCompleted(new FileCompletedEvent( + operation, releaseVersion, targetGroupAlias, targetAlias, fileName, migrationRecordId, outcome, DateTime.UtcNow - startedAtUtc, errorMessage))); + + private void ReportBlockCompleted(MigrationOperation operation, MigrationFileInfo file, string targetAlias, int migrationRecordId, int blockNumber, bool success, string? errorMessage = null) + => Report(nameof(IMigrationProgressObserver.OnBlockCompleted), o => o.OnBlockCompleted(new BlockCompletedEvent( + operation, file.ReleaseVersion, file.TargetGroupAlias, targetAlias, file.Filename, migrationRecordId, blockNumber, file.FileUpBlocksTotal, success, errorMessage))); + + private void ReportBlockCompleted(MigrationOperation operation, MigrationRecord record, int blocksTotal, int blockNumber, bool success, string? errorMessage) + => Report(nameof(IMigrationProgressObserver.OnBlockCompleted), o => o.OnBlockCompleted(new BlockCompletedEvent( + operation, record.ReleaseVersion, record.TargetGroupAlias, record.TargetAlias, record.Filename, record.Id, blockNumber, blocksTotal, success, errorMessage))); + + #endregion Progress events + /// /// The execution path reads the run mode from the request, while every repository stamp and the /// DatabaseLogging gate read it from the . The CLI fills both from the same @@ -1121,6 +1248,28 @@ internal void LogAtomicSharedConnectionTargets( #region MigrateDown public async Task MigrateDownAsync(MigrateDownRequest request) + { + _runStartedReported = false; + var runStart = DateTime.UtcNow; + MigrationOperationResult result; + try + { + result = await MigrateDownCoreAsync(request); + } + catch (Exception ex) + { + ReportRunCompleted(MigrationCommand.MigrateDown, request.RunMode, request.TargetReleaseVersion, + false, null, 0, 0, 0, DateTime.UtcNow - runStart, ex.Message); + throw; + } + + ReportRunCompleted(MigrationCommand.MigrateDown, request.RunMode, request.TargetReleaseVersion, + result.Success, result.Result, result.TotalMigrations, result.SuccessfulMigrations, result.FailedMigrations, + result.Duration, result.ErrorMessage); + return result; + } + + private async Task MigrateDownCoreAsync(MigrateDownRequest request) { var startTime = DateTime.UtcNow; @@ -1216,6 +1365,8 @@ public async Task MigrateDownAsync(MigrateDownRequest await RepositoryMigrationRunInsertWithAutoFix(settingsJson); } + ReportRunStarted(MigrationCommand.MigrateDown, request.RunMode, request.TargetReleaseVersion); + // --- Phase 2: Query migrations for rollback --- // RepositoryMigrationSelect always reads Migrate-mode records (the only ones that exist). List existingRecords; @@ -1376,6 +1527,27 @@ public async Task MigrateDownAsync(MigrateDownRequest #region Baseline public async Task BaselineAsync(BaselineRequest request) + { + _runStartedReported = false; + var runStart = DateTime.UtcNow; + BaselineResult result; + try + { + result = await BaselineCoreAsync(request); + } + catch (Exception ex) + { + ReportRunCompleted(MigrationCommand.Baseline, MigrationRunMode.Migrate, request.TargetReleaseVersion, + false, null, 0, 0, 0, DateTime.UtcNow - runStart, ex.Message); + throw; + } + + ReportRunCompleted(MigrationCommand.Baseline, MigrationRunMode.Migrate, request.TargetReleaseVersion, + result.Success, null, result.BaselinedFiles, result.BaselinedFiles, 0, result.Duration, result.ErrorMessage); + return result; + } + + private async Task BaselineCoreAsync(BaselineRequest request) { var startTime = DateTime.UtcNow; @@ -1409,6 +1581,7 @@ public async Task BaselineAsync(BaselineRequest request) var settingsJson = BuildMigrationRunSettingsJson(_ctxAccessor.Current); _logger.LogTrace("MigrationRun settings snapshot:\n{SettingsJson}", settingsJson); await RepositoryMigrationRunInsertWithAutoFix(settingsJson); + ReportRunStarted(MigrationCommand.Baseline, MigrationRunMode.Migrate, request.TargetReleaseVersion); // --- Phase 2: File Discovery --- var productOptions = _ctxAccessor.Current.RayMigratorOptions.Products!.First(p => p.Alias == request.ProductAlias); @@ -1548,6 +1721,9 @@ async Task BaselineFile(MigrationFileInfo file, TargetOptions targetOptions) return; } + var baselineStart = DateTime.UtcNow; + ReportFileStarted(MigrationOperation.Baseline, file, targetOptions.Alias!, 0, 0, baselineStart); + // Validate CLI tool alias if set (no execution, but ensures config is correct for future rollbacks) string? cliAlias = ResolveUseCliToolAlias(file, targetOptions); if (cliAlias != null) @@ -1580,6 +1756,7 @@ async Task BaselineFile(MigrationFileInfo file, TargetOptions targetOptions) await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( migrationRecordId, MigrationStatus.Migrated, file.FileUpBlocksTotal)); + ReportFileCompleted(MigrationOperation.Baseline, file, targetOptions.Alias!, migrationRecordId, FileOutcome.Baselined, baselineStart); _logger.LogDebug("Baselined: {Filename} (Release: {Release}, Target: {Target})", file.Filename, file.ReleaseVersion, targetOptions.Alias); @@ -1700,6 +1877,10 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( record.Id, MigrationStatus.Failed, record.FileUpBlocksMigrated)); } + ReportFileCompleted(RollbackOperation(isErrorRecovery), record.ReleaseVersion, record.TargetGroupAlias, record.TargetAlias, record.Filename, + + record.Id, FileOutcome.RollbackFailed, fileStartTime, $"Required rollback file not found: {rollbackFilename}"); + result.AddFailure(record.Filename, $"Required rollback file not found: {rollbackFilename}", record.TargetAlias, record.Id); result.ErrorMessage = $"Rollback chain aborted: required rollback file missing for {record.Filename}"; return result; @@ -1779,6 +1960,9 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdateRollback( rollbackFileInfo.FileUpConfigJson)); } + ReportFileStarted(RollbackOperation(isErrorRecovery), record.ReleaseVersion, record.TargetGroupAlias, record.TargetAlias, + record.Filename, record.FileOrderId, record.Id, rollbackFileInfo.FileUpBlocksTotal, 0, fileStartTime); + // Execute rollback SQL blocks bool fileHadBlockError = false; bool atomicRollbackDone = false; @@ -1846,7 +2030,7 @@ await ExecuteWithCliTool( try { await ExecuteRollbackBlocksAtomic( - rollbackFileInfo, targetDal!, dalSettings, record, runMode, rollbackStartBlock); + rollbackFileInfo, targetDal!, dalSettings, record, runMode, rollbackStartBlock, RollbackOperation(isErrorRecovery)); atomicRollbackDone = true; } catch (Exception blockEx) @@ -1870,6 +2054,10 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdateRollback( rollbackFileInfo.FileUpConfigJson)); } + ReportFileCompleted(RollbackOperation(isErrorRecovery), record.ReleaseVersion, record.TargetGroupAlias, record.TargetAlias, record.Filename, + + record.Id, FileOutcome.RollbackFailed, fileStartTime, $"Atomic rollback failed: {blockEx.Message}"); + result.AddFailure(record.Filename, $"Atomic rollback failed: {blockEx.Message}", record.TargetAlias, record.Id); result.ErrorMessage = $"Rollback chain aborted: atomic rollback of {record.Filename} failed"; @@ -1905,6 +2093,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdateRollback( blockIndex + 1, rollbackFileInfo.FileUpBlocksTotal, rollbackFileInfo.FileUpConfigJson)); + ReportBlockCompleted(RollbackOperation(isErrorRecovery), record, rollbackFileInfo.FileUpBlocksTotal, blockIndex + 1, true, null); } catch (Exception blockEx) { @@ -1925,6 +2114,10 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdateRollback( rollbackFileInfo.FileUpBlocksTotal, rollbackFileInfo.FileUpConfigJson)); + ReportFileCompleted(RollbackOperation(isErrorRecovery), record.ReleaseVersion, record.TargetGroupAlias, record.TargetAlias, record.Filename, + + record.Id, FileOutcome.RollbackFailed, fileStartTime, $"Rollback block {blockIndex + 1} failed: {blockEx.Message}"); + result.AddFailure(record.Filename, $"Rollback block {blockIndex + 1} failed: {blockEx.Message}", record.TargetAlias, record.Id); result.ErrorMessage = $"Rollback chain aborted: block {blockIndex + 1} of {record.Filename} failed"; @@ -1937,6 +2130,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdateRollback( "Rollback block {Block}/{Total} failed for migration {MigrationRecordId} ({Filename}). RollbackErrorAction=Ignore — skipping block, continuing rollback.", blockIndex + 1, rollbackFileInfo.FileUpBlocksTotal, record.Id, record.Filename); + ReportBlockCompleted(RollbackOperation(isErrorRecovery), record, rollbackFileInfo.FileUpBlocksTotal, blockIndex + 1, false, blockEx.Message); fileHadBlockError = true; } } @@ -1987,6 +2181,10 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdateRollback( result.SuccessCount++; } + ReportFileCompleted(RollbackOperation(isErrorRecovery), record.ReleaseVersion, record.TargetGroupAlias, record.TargetAlias, record.Filename, + record.Id, fileHadBlockError ? FileOutcome.RollbackFailed : FileOutcome.RolledBack, fileStartTime, + fileHadBlockError ? "Rollback completed with errors (some blocks failed, RollbackErrorAction=Ignore)" : null); + result.FileResults.Add(new MigrationFileResult { FileName = record.Filename, @@ -2010,6 +2208,10 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdateRollback( "CRITICAL: Rollback failed for migration {MigrationRecordId} ({Filename}). Rollback chain ABORTED.", record.Id, record.Filename); + ReportFileCompleted(RollbackOperation(isErrorRecovery), record.ReleaseVersion, record.TargetGroupAlias, record.TargetAlias, record.Filename, + + record.Id, FileOutcome.RollbackFailed, fileStartTime, ex.Message); + result.AddFailure(record.Filename, ex.Message, record.TargetAlias, record.Id); result.ErrorMessage = $"Rollback chain aborted at {record.Filename}: {ex.Message}"; @@ -3012,6 +3214,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( } succeededBlocks++; + ReportBlockCompleted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, migrationRecordId, i + 1, success: true); } return (succeededBlocks, failedBlocks, false); } @@ -3051,6 +3254,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( string sqlBlock = ReplaceEnvironmentVariablesInSqlBlock( file.SqlBlocks[blockIndex], file.Filename, blockIndex + 1, file.FileUpBlocksTotal); _ctxAccessor.Current.MigrationState.FileBlockId = blockIndex + 1; + bool blockSucceeded = true; + string? blockError = null; _logger.LogDebug("Executing block {Block}/{Total} from {Filename} against target {Target}", blockIndex + 1, file.FileUpBlocksTotal, file.Filename, targetOptions.Alias); @@ -3070,6 +3275,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( catch (Exception blockEx) { failedBlocks++; + blockSucceeded = false; + blockError = blockEx.Message; _logger.LogWarning(blockEx, "MigrationErrorAction=Ignore: Block {Block}/{Total} from {Filename} failed on target {Target}. Continuing with next block.", blockIndex + 1, file.FileUpBlocksTotal, file.Filename, targetOptions.Alias); @@ -3085,6 +3292,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( // Update block progress in repository await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( migrationRecordId, MigrationStatus.Executing, blockIndex + 1)); + ReportBlockCompleted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, migrationRecordId, blockIndex + 1, blockSucceeded, blockError); } return (succeededBlocks, failedBlocks, false); @@ -3147,6 +3355,7 @@ await targetDal.ExecuteNonQueryAsync( _templateExecutor.RepositoryMigrationUpdate( migrationRecordId, MigrationStatus.Executing, blockIndex + 1, connection, transaction, repoTimeout); + ReportBlockCompleted(MigrationOperation.MigrateUp, file, _ctxAccessor.Current.MigrationState.TargetAlias, migrationRecordId, blockIndex + 1, success: true); } // Final status update INSIDE the transaction (this is what closes the atomicity gap) @@ -3192,7 +3401,8 @@ private async Task ExecuteRollbackBlocksAtomic( DalSettings dalSettings, MigrationRecord record, MigrationRunMode runMode, - int rollbackStartBlock) + int rollbackStartBlock, + MigrationOperation operation) { int repoTimeout = (int)_ctxAccessor.Current.RayMigratorOptions.Repository!.DbCommandTimeoutInSeconds!; int maxRetries = dalSettings.MaxRetries; @@ -3238,6 +3448,7 @@ await targetDal.ExecuteNonQueryAsync( rollbackFileInfo.FileUpBlocksTotal, rollbackFileInfo.FileUpConfigJson, connection, transaction, repoTimeout); + ReportBlockCompleted(operation, record, rollbackFileInfo.FileUpBlocksTotal, blockIndex + 1, true, null); } // Final status update INSIDE the transaction (this is what closes the atomicity gap) diff --git a/Raycoon.RayMigrator.Tests.Engine/Infrastructure/RecordingProgressObserver.cs b/Raycoon.RayMigrator.Tests.Engine/Infrastructure/RecordingProgressObserver.cs new file mode 100644 index 0000000..58a90e8 --- /dev/null +++ b/Raycoon.RayMigrator.Tests.Engine/Infrastructure/RecordingProgressObserver.cs @@ -0,0 +1,23 @@ +using Raycoon.RayMigrator.Core.Progress; + +namespace Raycoon.RayMigrator.Tests.Engine.Infrastructure; + +/// +/// Records every progress event in the order it was raised, so that a test can assert the sequence. +/// +public sealed class RecordingProgressObserver : IMigrationProgressObserver +{ + public List Events { get; } = []; + + public IReadOnlyList Of() => Events.OfType().ToList(); + + public void OnRunStarted(RunStartedEvent e) => Events.Add(e); + + public void OnFileStarted(FileStartedEvent e) => Events.Add(e); + + public void OnBlockCompleted(BlockCompletedEvent e) => Events.Add(e); + + public void OnFileCompleted(FileCompletedEvent e) => Events.Add(e); + + public void OnRunCompleted(RunCompletedEvent e) => Events.Add(e); +} diff --git a/Raycoon.RayMigrator.Tests.Engine/Infrastructure/ScenarioContext.cs b/Raycoon.RayMigrator.Tests.Engine/Infrastructure/ScenarioContext.cs index d00c480..aa20a57 100644 --- a/Raycoon.RayMigrator.Tests.Engine/Infrastructure/ScenarioContext.cs +++ b/Raycoon.RayMigrator.Tests.Engine/Infrastructure/ScenarioContext.cs @@ -2,6 +2,7 @@ using Raycoon.RayMigrator.Core; using Raycoon.RayMigrator.Core.Configuration; using Raycoon.RayMigrator.Core.Configuration.Enums; +using Raycoon.RayMigrator.Core.Progress; using Raycoon.RayMigrator.Services.Abstractions; using Raycoon.RayMigrator.Testing; @@ -15,6 +16,7 @@ namespace Raycoon.RayMigrator.Tests.Engine.Infrastructure; public class ScenarioContext : IAsyncDisposable { private EngineTestHost _host; + private IMigrationProgressObserver? _progressObserver; private readonly RepositoryQueryHelper _queryHelper; private readonly EngineConfig _engineConfig; private readonly string _workDir; @@ -244,6 +246,15 @@ public void InsertOrphanedMigrationRun(int minutesOld = 120) /// public Core.Configuration.Options.RayMigratorConsoleOptions HostConsoleOptions => _host.MigrationContext.RayMigratorConsoleOptions; + /// + /// Sets the progress observer of the host's context, and of every context a later rebuild creates. + /// + public void UseProgressObserver(IMigrationProgressObserver observer) + { + _progressObserver = observer; + _host.MigrationContext.ProgressObserver = observer; + } + /// /// Rebuilds the DI container for a different command/mode without cleaning databases. /// Used for multi-step test scenarios (e.g., Migrate-Up then Migrate-Down). @@ -254,6 +265,11 @@ public Task RebuildForAsync(MigrationCommand command, MigrationRunMode mode, str _host.Dispose(); _host = new EngineTestHost(); _host.Build(_configPath, _productAlias, command, mode, toRelease, _environment); + if (_progressObserver != null) + { + _host.MigrationContext.ProgressObserver = _progressObserver; + } + return Task.CompletedTask; } diff --git a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqliteProgressEventsTests.cs b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqliteProgressEventsTests.cs new file mode 100644 index 0000000..fc2dd11 --- /dev/null +++ b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqliteProgressEventsTests.cs @@ -0,0 +1,176 @@ +using Raycoon.RayMigrator.Core.Configuration.Enums; +using Raycoon.RayMigrator.Core.Progress; +using Raycoon.RayMigrator.Tests.Engine.Fixtures; +using Raycoon.RayMigrator.Tests.Engine.Infrastructure; + +namespace Raycoon.RayMigrator.Tests.Engine.Tests.Features; + +/// +/// The typed progress events a host observes through : one run +/// event pair per operation, one file event pair per file and target, one block event per executed block. +/// +[Collection("Sqlite")] +[Trait("Engine", "Sqlite")] +[Trait("Category", "Features")] +public class SqliteProgressEventsTests : SqliteTestBase +{ + public SqliteProgressEventsTests(SqliteFixture fixture) : base(fixture) { } + + [Fact] + public async Task MigrateUp_ReportsTheRunAndEveryFileOnEveryTarget() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.MigrateUpAsync("Release_1.0"); + ctx.AssertSuccess(true); + + recorder.Events.First().Should().BeOfType(); + recorder.Events.Last().Should().BeOfType(); + + var started = recorder.Of().Should().ContainSingle().Which; + started.Command.Should().Be(MigrationCommand.MigrateUp); + started.RunMode.Should().Be(MigrationRunMode.Migrate); + started.MigrationRunId.Should().Be(result.MigrationRunId, "the run row exists when the run is reported as started"); + started.MigrationRunId.Should().BePositive(); + started.FromRelease.Should().BeNull("nothing was migrated before the first run"); + started.TargetRelease.Should().Be("Release_1.0"); + + var completed = recorder.Of().Should().ContainSingle().Which; + completed.Success.Should().BeTrue(); + completed.Result.Should().Be(MigrationRunResult.Ok); + completed.MigrationRunId.Should().Be(result.MigrationRunId); + completed.TotalFiles.Should().Be(result.TotalMigrations); + + var fileStarts = recorder.Of(); + var fileEnds = recorder.Of(); + fileStarts.Should().HaveCount(result.TotalMigrations, "one start per file and target"); + fileEnds.Should().HaveCount(result.TotalMigrations, "one completion per file and target"); + fileEnds.Should().OnlyContain(e => e.Outcome == FileOutcome.Migrated && e.Operation == MigrationOperation.MigrateUp); + fileStarts.Should().OnlyContain(e => e.MigrationRecordId > 0 && e.ReleaseVersion == "Release_1.0" && e.BlocksTotal >= 1); + fileEnds.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)) + .Should().BeEquivalentTo(fileStarts.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), + "a completion carries the same identity as its start"); + + foreach (var end in fileEnds) + { + int startIndex = recorder.Events.IndexOf(fileStarts.Single(s => s.FileName == end.FileName && s.TargetAlias == end.TargetAlias)); + recorder.Events.IndexOf(end).Should().BeGreaterThan(startIndex, "a file completes after it started"); + } + + var blocks = recorder.Of(); + blocks.Should().NotBeEmpty(); + blocks.Should().OnlyContain(b => b.Success && b.BlockNumber >= 1 && b.BlockNumber <= b.BlocksTotal); + blocks.Select(b => b.FileName).Distinct().Should().BeEquivalentTo(fileStarts.Select(s => s.FileName).Distinct(), + "every executed file reports its blocks"); + } + + [Fact] + public async Task Simulate_ReportsSimulatedOutcomesWithoutARunRow() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(MigrationCommand.MigrateUp, MigrationRunMode.Simulate); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + await ctx.MigrateUpAsync(runMode: MigrationRunMode.Simulate); + ctx.AssertSuccess(true); + + recorder.Of().Should().ContainSingle().Which.MigrationRunId.Should().Be(0, "Simulate writes no run row"); + recorder.Of().Should().NotBeEmpty().And.OnlyContain(e => e.Outcome == FileOutcome.Simulated); + recorder.Of().Should().ContainSingle().Which.Success.Should().BeTrue(); + } + + [Fact] + public async Task MigrateDown_ReportsRollbacksOfTheRecordsAboveTheTargetRelease() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + await ctx.RebuildForAsync(MigrationCommand.MigrateDown, MigrationRunMode.Migrate, "Release_2.0"); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.MigrateDownAsync("Release_2.0"); + ctx.AssertSuccess(true); + + recorder.Of().Should().ContainSingle().Which.Command.Should().Be(MigrationCommand.MigrateDown); + var ends = recorder.Of(); + ends.Should().HaveCount(result.TotalMigrations); + ends.Should().OnlyContain(e => e.Operation == MigrationOperation.MigrateDown && e.Outcome == FileOutcome.RolledBack && e.MigrationRecordId > 0); + ends.Select(e => e.ReleaseVersion).Distinct().Should().BeEquivalentTo(["Release_3.0", "Release_4.0"]); + recorder.Of().Should().HaveCount(ends.Count); + recorder.Of().Should().ContainSingle().Which.Success.Should().BeTrue(); + } + + [Fact] + public async Task Baseline_ReportsBaselinedFilesWithoutBlocks() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(MigrationCommand.Baseline); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.BaselineAsync(); + ctx.AssertSuccess(true); + + recorder.Of().Should().ContainSingle().Which.Command.Should().Be(MigrationCommand.Baseline); + recorder.Of().Should().HaveCount(result.BaselinedFiles) + .And.OnlyContain(e => e.Operation == MigrationOperation.Baseline && e.Outcome == FileOutcome.Baselined && e.MigrationRecordId > 0); + recorder.Of().Should().BeEmpty("baseline executes no SQL"); + recorder.Of().Should().ContainSingle().Which.TotalFiles.Should().Be(result.BaselinedFiles); + } + + [Fact] + public async Task FailingFile_ReportsTheFailedFileAndTheFailedRun() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario() + .InjectError("Release_2.0", "02_CreateTableD.sql") + .BuildAsync(); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.MigrateUpAsync(); + ctx.AssertSuccess(false); + + var failed = recorder.Of().Where(e => e.Outcome == FileOutcome.Failed).Should().ContainSingle().Which; + failed.FileName.Should().Be("02_CreateTableD.sql"); + failed.ReleaseVersion.Should().Be("Release_2.0"); + failed.ErrorMessage.Should().NotBeNullOrEmpty(); + failed.MigrationRecordId.Should().BePositive(); + recorder.Of().Where(e => e.Outcome == FileOutcome.Migrated).Should().NotBeEmpty("the files before the broken one ran"); + + var completed = recorder.Of().Should().ContainSingle().Which; + completed.Success.Should().BeFalse(); + completed.Result.Should().Be(result.Result); + completed.FailedFiles.Should().Be(result.FailedMigrations); + recorder.Events.Last().Should().BeSameAs(completed, "the run completion is the last event"); + } + + [Fact] + public async Task ObserverThatThrows_DoesNotAbortTheRun() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + ctx.UseProgressObserver(new ThrowingProgressObserver()); + + await ctx.MigrateUpAsync("Release_1.0"); + + ctx.AssertSuccess(true); + ctx.AssertRunResult(MigrationRunResult.Ok); + } + + private sealed class ThrowingProgressObserver : IMigrationProgressObserver + { + public void OnRunStarted(RunStartedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnFileStarted(FileStartedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnBlockCompleted(BlockCompletedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnFileCompleted(FileCompletedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnRunCompleted(RunCompletedEvent e) => throw new InvalidOperationException("observer failure"); + } +} From 3276bf6a4c03d46ba89bff87528939e2fb67c132 Mon Sep 17 00:00:00 2001 From: danielpavic Date: Mon, 12 Oct 2026 22:15:12 +0200 Subject: [PATCH 06/14] Document the hosting seam, the options preparer and the progress events The MigrationContext chapter gains a "Hosting the engine in-process" section with the per-run sequence a host follows, the per-process pieces, the process-global state and the progress observer; the factory section shows the new contract. The service chapter shows the constructor without IOptions, the progress events and the registration with the singleton accessor; the configuration chapter describes RayMigratorOptionsPreparer in place of the "pre-built options are trusted" paragraph; the DI chapter, the template-executor chapter and the coding conventions follow. ADR-023 records the decision, and CLAUDE.md names the seam. --- CLAUDE.md | 3 +- Docs/01-architecture/dependency-injection.md | 2 +- Docs/02-core-concepts/configuration-system.md | 5 +- Docs/02-core-concepts/migration-context.md | 90 +++++++++++-------- Docs/04-service-layer/migration-service.md | 40 ++++++--- Docs/04-service-layer/template-executor.md | 8 +- Docs/11-development/coding-conventions.md | 4 +- Docs/arc42/09-Architecture-Decisions.md | 13 +++ 8 files changed, 104 insertions(+), 61 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8c8e776..d183d4a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -66,7 +66,8 @@ Seven commands, lowercase kebab-case: `migrate-up`, `migrate-down`, `validate-ha - **Configuration** sits under the `RayMigrator` root section: `Repository`, `ProductDefaults` → `TargetGroupDefaults` → `TargetDefaults`, `Products` → `TargetGroups` → `Targets`, `DatabaseLogging`, `CliTools`, `Serilog`. The target-group order option is `TargetMigrationOrder` (`FileByFile` / `TargetByTarget`); `TargetGroupMigrationOrder` on a product orders the groups. Arrays whose elements carry an `Alias` merge by alias across the `appsettings*.json` hierarchy (ADR-021). - **Placeholders**: `{ENV:VAR}` in configuration and CLI values, `{CFG:SchemaName}` / `{CFG:TableBaseName}` in SQL templates. Runtime values are ADO parameters. - **SQL templates** are `Content` files, not embedded resources. Result contract is `SELECT 'code[,code...],message'`; the first code comes from `Shared/Constants/TemplateResultCode.cs`, further integer codes are template-specific. -- **Options validation** uses DataAnnotations plus `Ray*Attribute`s in Core; enum options are strings resolved through `ParsedEnumOption`. Cross-field rules live in the `Validation` project and are shared with the wizard. +- **Options validation** uses DataAnnotations plus `Ray*Attribute`s in Core; enum options are strings resolved through `ParsedEnumOption`. Cross-field rules live in the `Validation` project and are shared with the wizard. Options built in memory get the same treatment from `RayMigratorOptionsPreparer`. +- **Hosting seam**: a host creates one `MigrationContext` per run through `IMigrationContextFactory` (`Services`), which prepares and activates it; `MigrationService` reads the configuration from the context, not from `IOptions`. Progress goes through `MigrationContext.ProgressObserver`. See `Docs/02-core-concepts/migration-context.md`. - **Docs and repo**: `CLAUDE.md` is tracked. `.claude/` (skills, agents, settings) and `CLAUDE.local.md` are local and gitignored. `Docs/arc42/` is mirrored to the GitHub Wiki on every push to `main`. - **Release** follows [Docs/11-development/release-process.md](Docs/11-development/release-process.md). Do not tag or push to `main` outside that process. diff --git a/Docs/01-architecture/dependency-injection.md b/Docs/01-architecture/dependency-injection.md index b2b70f3..c1c8f43 100644 --- a/Docs/01-architecture/dependency-injection.md +++ b/Docs/01-architecture/dependency-injection.md @@ -127,7 +127,7 @@ Services created per operation: ### Options Pattern -Configuration sections are bound to strongly-typed options classes with DataAnnotations validation: +Configuration sections are bound to strongly-typed options classes with DataAnnotations validation. The binding is the CLI pipeline's way of loading and validating the files; the services never consume `IOptions`. They read the configuration of the current run from `MigrationContext.RayMigratorOptions`, which the context factory receives from the pipeline (`CreateFromValidatedOptions`) or prepares itself for a host that built the options in memory (`Create`, see [MigrationContext](../02-core-concepts/migration-context.md#hosting-the-engine-in-process)). ```csharp // Registration with validation (JSON mode) diff --git a/Docs/02-core-concepts/configuration-system.md b/Docs/02-core-concepts/configuration-system.md index 136aab2..85084d8 100644 --- a/Docs/02-core-concepts/configuration-system.md +++ b/Docs/02-core-concepts/configuration-system.md @@ -274,7 +274,7 @@ sequenceDiagram ### ManagedLocal Mode (Admin-DB) -> **Implementation Status**: The `OperatingMode.ManagedLocal` enum value and `AdminDbOptions` configuration class exist in Engine's Core layer (part of the NuGet contract), but the Admin-DB operating mode is implemented in **RayMigrator Studio**, not in Engine. When Studio uses ManagedLocal mode, it supplies a pre-built `RayMigratorOptions` via the `PreBuiltOptions` field of `OptionsSourceResult`, and the `DirectModePipeline` handles the rest of the lifecycle identically to Standalone mode. +> **Implementation Status**: The `OperatingMode.ManagedLocal` enum value and `AdminDbOptions` configuration class exist in Engine's Core layer (part of the NuGet contract), but the Admin-DB operating mode is implemented in **RayMigrator Studio**, not in Engine. When Studio uses ManagedLocal mode, it supplies a pre-built `RayMigratorOptions` via the `PreBuiltOptions` field of `OptionsSourceResult`, and the `DirectModePipeline` handles the rest of the lifecycle identically to Standalone mode. A host that runs the engine in-process creates its context through `IMigrationContextFactory.Create`, which prepares in-memory options with `RayMigratorOptionsPreparer` (see [Validation](#validation)). ## Options Source Abstraction @@ -302,11 +302,12 @@ Options are validated at startup via three layered mechanisms, wired up in `Dire Warnings (non-blocking issues from the shared catalog) are emitted to the static `Serilog.Log` channel at `Warning` level before validation decides pass/fail; only errors cause `host.Build()` to throw. -**Admin-DB / Studio mode**: the `PreBuiltOptions` branch in `DirectModePipeline` (used by RayMigrator-Studio's API layer) registers `RayMigratorOptions` via `Options.Create(...)` and therefore does **not** invoke `IValidateOptions`. This is intentional — pre-built options from Studio's `AdminDbOptionsSource` are trusted. If future Studio code needs validation, it should call `RayMigratorOptionsValidator.Validate(null, preBuilt)` explicitly. +**Options built in memory** (a host such as RayMigrator Studio, or a test): `RayMigratorOptionsPreparer.Prepare(options)` (namespace `Raycoon.RayMigrator.Core.Configuration.Validation`) gives an options object the same treatment, in the same order, without the options framework: `{ENV:VARIABLE}` placeholders are resolved in every string of the graph (`EnvironmentVariableReplacer.ReplaceWithEnvironmentVariables(RayMigratorOptions)`; a missing variable throws and names the variable and the path), `ProductDefaultsPostConfigureOptions` merges the defaults and probes the enum values, `DataAnnotationValidateOptions` runs the attributes and writes the `RayRangeInt` defaults back, and `RayMigratorOptionsValidator` applies the rule catalog. Every finding of the failing step is reported in one `ConfigurationValidationException`. The resolved values and the configuration's secrets are registered with `SensitiveDataMasker`. `IMigrationContextFactory.Create` calls the preparer; `CreateFromValidatedOptions` skips it for options the options framework already validated. The `PreBuiltOptions` branch of `DirectModePipeline` still runs only the enum probe before it registers the options with `Options.Create(...)`; it exists for the CLI's ManagedLocal mode and is not the path a host should take. Namespaces: - Custom attributes: `Raycoon.RayMigrator.Core.Configuration.Validation.RayAttributes` - Engine-side adapter: `Raycoon.RayMigrator.Core.Configuration.Validation.OptionsValidationInputAdapter` +- In-memory preparation: `Raycoon.RayMigrator.Core.Configuration.Validation.RayMigratorOptionsPreparer` - Shared catalog: `Raycoon.RayMigrator.Validation` (zero dependencies, WASM-safe) For the authoritative list of all rule codes, their severities, and example messages, see [Appendix: Validation Rules](../appendix/validation-rules.md). For a broader overview of where validation attributes sit in the configuration hierarchy, see [Configuration Hierarchy — Validation](../06-configuration-reference/appsettings-hierarchy.md). diff --git a/Docs/02-core-concepts/migration-context.md b/Docs/02-core-concepts/migration-context.md index 1ea4fee..c209d5c 100644 --- a/Docs/02-core-concepts/migration-context.md +++ b/Docs/02-core-concepts/migration-context.md @@ -156,71 +156,82 @@ public class AsyncLocalMigrationContextAccessor : IMigrationContextAccessor ### `IMigrationContextFactory` / `MigrationContextFactory` -Factory for creating `MigrationContext` instances. CLI creates one at startup; API creates one per request. The `command` is stamped into the MigrationRun settings snapshot and drives the command's side effects (`CommandProfile`), so it must be the command that is actually executed (#6). `runMode` is only a choice for `MigrateUp` / `MigrateDown`; pass `MigrationRunMode.Migrate` for every other command. +The factory is the entry point of every host for one run, the CLI pipeline included. The interface lives in `Core`, the implementation in `Services` (it needs `DalFactory`) and is registered by `AddRayMigratorServices`. The console options carry the command, product, environment, run mode, target release and flags; the `command` is stamped into the MigrationRun settings snapshot and drives the command's side effects (`CommandProfile`), so it must be the command that is actually executed (#6). `runMode` is only a choice for `MigrateUp` / `MigrateDown`; pass `MigrationRunMode.Migrate` for every other command. ```csharp public interface IMigrationContextFactory { - MigrationContext Create( - RayMigratorOptions options, string product, string environment, - MigrationCommand command, MigrationRunMode runMode, string version, - string? targetReleaseVersion = null, bool revealSensitiveData = false); -} + // Options a host built in memory: prepared with RayMigratorOptionsPreparer first + MigrationContext Create(RayMigratorOptions options, RayMigratorConsoleOptions consoleOptions); -public class MigrationContextFactory : IMigrationContextFactory -{ - public MigrationContext Create( - RayMigratorOptions options, string product, string environment, - MigrationCommand command, MigrationRunMode runMode, string version, - string? targetReleaseVersion = null, bool revealSensitiveData = false) - { - var consoleOptions = new RayMigratorConsoleOptions - { - Command = command, - Product = product, - Environment = environment, - RunMode = runMode, - TargetReleaseVersion = targetReleaseVersion, - ShowStartupInfo = false, - RevealSensitiveData = revealSensitiveData, - }; - return new MigrationContext(options, consoleOptions, version); - } + // Options the options framework already bound, merged and validated (the CLI pipeline) + MigrationContext CreateFromValidatedOptions(RayMigratorOptions options, RayMigratorConsoleOptions consoleOptions); } ``` +Both methods do the same work after the preparation step: + +1. Build the context. An unknown product alias throws a `ConfigurationValidationException` whose message names a near miss or the configured aliases (`ProductAliasGuard`). +2. Fill `DalSpecificPropertiesDictionary` for the repository, every target of the product and the database logging, and check the schema names against the DAL capabilities. A host that skipped this step fell back to the SQL Server block delimiter `GO` for every engine. +3. Register the configuration's secrets with `SensitiveDataMasker` and make the context current: `IMigrationContextAccessor.Current` and `MigrationLoggingContext.Current` of the calling async flow. + +`Create` additionally runs `RayMigratorOptionsPreparer.Prepare`, which gives in-memory options what options bound from `appsettings*.json` always got: `{ENV:VARIABLE}` placeholders are resolved in every string of the options graph (a missing variable is an error that names the variable and the path), the product defaults are merged down, every enum-typed value is probed, the data annotations run and write their documented defaults into null properties, and the rule catalog is applied. Preparing an object twice is harmless. + +### Hosting the engine in-process + +A host (RayMigrator Studio, an integration test) runs one operation per context. The sequence per run is: + +1. Build the options object, or load it from a store, and the `RayMigratorConsoleOptions` of the operation. +2. Open the masking scope for the async flow of the run: `using var scope = SensitiveDataMasker.BeginScope(revealSensitiveData);`. Without an active scope or global initialization the masker is off and the MigrationRun settings snapshot stores connection strings unmasked. +3. `factory.Create(options, consoleOptions)`. The returned context is current for the calling async flow; set `context.ProgressObserver` before the operation if the host wants progress events. +4. Optionally `ConnectionValidator.ValidateTargetConnections(context, logger)`: it opens the connections the command's profile needs and fails fast with a clear message. +5. Call the operation on a scoped `IMigrationService`. The request must agree with the context: `MigrateUpAsync`, `MigrateDownAsync` and `BaselineAsync` refuse a request whose run mode or target release differs from the console options of the context (#6, #17). +6. Dispose the scope. The next run gets a fresh scope, a fresh context and a fresh `TemplateExecutor` (it caches the repository of the first context it sees, so it is registered as scoped). + +Per process, not per run: one `TemplateCache` (`new TemplateCache(revealSensitiveData, logger)`, then `ValidateConfigurationAgainstTemplateCache(options)` for each configuration the host runs), the `DatabaseLogWriter` and its Serilog sink when database logging is wanted, and the Serilog logger with `MigrationContextEnricher`, which reads `MigrationLoggingContext.Current` and stamps every event with run, release, target group, target, file and block. + +Process-global state a host shares between runs: `SensitiveDataMasker` (per-flow scopes isolate runs; `Initialize` sets the global mode the CLI uses), `MigrationLoggingContext` and `AsyncLocalMigrationContextAccessor` (both `AsyncLocal`, isolated per async flow), `DalFactory` (type map and DAL instances per database type and connection string, safe to share). The repository lock allows one run per product and environment; runs of different combinations may execute in parallel in separate scopes and async flows. `DirectModePipeline` stays CLI-only: it replaces the static `Log.Logger`, writes to the console and returns an exit code. + +### `ProgressObserver` + +`MigrationContext.ProgressObserver` receives the lifecycle events of the run through `IMigrationProgressObserver` (namespace `Raycoon.RayMigrator.Core.Progress`): `RunStartedEvent` after the run row exists (with its id), `FileStartedEvent` and `FileCompletedEvent` for every file on every target (migration, rollback and baseline, with the record id and a `FileOutcome`), `BlockCompletedEvent` for every executed SQL block, and `RunCompletedEvent` with the result. The engine raises them synchronously on the thread of the run; an observer that throws is logged and the run continues. The default `NullMigrationProgressObserver` ignores everything, and the CLI sets none. See [Migration Service: Progress events](../04-service-layer/migration-service.md#progress-events). + ### `RayMigratorHostMode` -Enum controlling DI registration behavior: +Enum controlling which accessor `AddRayMigratorServices` registers. Both accessors are singletons: the CLI holds one context for the whole process, and the API accessor isolates runs per async flow through a static `AsyncLocal`, so a scoped registration would isolate nothing. ```csharp public enum RayMigratorHostMode { Cli, // SingletonMigrationContextAccessor - Api // AsyncLocalMigrationContextAccessor (Scoped) + Api // AsyncLocalMigrationContextAccessor (per async flow) } ``` ### API-Mode Lifecycle -> **Note**: The API hosting mode is implemented in **RayMigrator Studio** (a separate repository). The classes `AsyncLocalMigrationContextAccessor`, `MigrationContextFactory`, and `RayMigratorHostMode` are part of the Engine's Core NuGet contract and are consumed by Studio. The Engine itself always uses `RayMigratorHostMode.Cli` (singleton accessor). The following diagram illustrates the per-request lifecycle as implemented in Studio. +> **Note**: The API hosting mode is implemented in **RayMigrator Studio** (a separate repository). `AsyncLocalMigrationContextAccessor`, `IMigrationContextFactory` and `RayMigratorHostMode` are part of the Engine's NuGet contract and are consumed by Studio. The Engine itself always uses `RayMigratorHostMode.Cli` (singleton accessor). The following diagram illustrates the per-run lifecycle as a host implements it. ```mermaid sequenceDiagram - participant Client - participant Endpoint as API Endpoint + participant Host participant Factory as MigrationContextFactory participant Accessor as AsyncLocalMigrationContextAccessor participant Svc as MigrationService - - Client->>Endpoint: POST /api/v1/migrations/up - Endpoint->>Factory: Create(options, product, env, command, ...) - Factory-->>Endpoint: new MigrationContext - Endpoint->>Accessor: accessor.Current = ctx - - Endpoint->>Svc: MigrateUpAsync(request) - Svc->>Accessor: _ctxAccessor.Current (reads per-request context) - Svc-->>Endpoint: MigrationOperationResult + participant Observer as IMigrationProgressObserver + + Host->>Host: SensitiveDataMasker.BeginScope(...) + Host->>Factory: Create(options, consoleOptions) + Factory->>Factory: RayMigratorOptionsPreparer.Prepare, DAL properties, schema names + Factory->>Accessor: Current = ctx (and MigrationLoggingContext.Current) + Factory-->>Host: ctx + Host->>Host: ctx.ProgressObserver = observer + + Host->>Svc: MigrateUpAsync(request) + Svc->>Accessor: _ctxAccessor.Current (reads the context of this async flow) + Svc->>Observer: OnRunStarted, OnFileStarted, OnBlockCompleted, OnFileCompleted, ... + Svc-->>Host: MigrationOperationResult (MigrationRunId, per-file results) + Svc->>Observer: OnRunCompleted ``` ## Lifecycle @@ -404,6 +415,7 @@ The `MigrationContext` is designed for single-threaded execution within its scop 2. **Validate context early** in service methods 3. **Update state at boundaries** (before/after each operation) 4. **Never modify RayMigratorOptions directly** after context creation +5. **One context per operation** in a host; set `ProgressObserver` before the call, not during it ## Related Documentation diff --git a/Docs/04-service-layer/migration-service.md b/Docs/04-service-layer/migration-service.md index 15061c4..4a98208 100644 --- a/Docs/04-service-layer/migration-service.md +++ b/Docs/04-service-layer/migration-service.md @@ -253,7 +253,7 @@ Returned by `MigrateUpAsync` and `MigrateDownAsync`: ```csharp public class MigrationOperationResult : OperationResult { - public Guid RunId { get; set; } + public int MigrationRunId { get; set; } // MigrationRun.Id; 0 when the run mode writes no run row public string ProductAlias { get; set; } = string.Empty; public string Environment { get; set; } = string.Empty; public MigrationOperation Operation { get; set; } @@ -306,6 +306,8 @@ Returned by `BaselineAsync`. `TargetReleaseVersion` is nullable — `null` when public class BaselineResult : OperationResult { public string ProductAlias { get; set; } = string.Empty; + public string Environment { get; set; } = string.Empty; + public int MigrationRunId { get; set; } // MigrationRun.Id; 0 when the run failed before the row was inserted public string? TargetReleaseVersion { get; set; } public int BaselinedFiles { get; set; } } @@ -319,6 +321,7 @@ Returned by `GetStatusAsync`: public class MigrationStatusInfo { public string ProductAlias { get; set; } = string.Empty; + public string Environment { get; set; } = string.Empty; public string CurrentRelease { get; set; } = string.Empty; public DateTime? LastMigrationDate { get; set; } public int TotalMigrationsExecuted { get; set; } @@ -352,6 +355,7 @@ Returned by `GetHistoryAsync`: public class MigrationHistory { public string ProductAlias { get; set; } = string.Empty; + public string Environment { get; set; } = string.Empty; // the runs of this environment only public List Runs { get; set; } = new List(); } ``` @@ -364,16 +368,16 @@ Individual run entry within `MigrationHistory`: public class MigrationRunInfo { public int MigrationRunId { get; set; } - public Guid RunId { get; set; } + public string Environment { get; set; } = string.Empty; public DateTime StartedAt { get; set; } public DateTime? CompletedAt { get; set; } public MigrationOperation Operation { get; set; } public MigrationRunResult Result { get; set; } public MigrationRunMode RunMode { get; set; } - public string? InitiatedBy { get; set; } public int TotalMigrations { get; set; } public int SuccessfulMigrations { get; set; } public int FailedMigrations { get; set; } + public string? FromRelease { get; set; } // the highest Migrated release when the run started; null for a first run public string? ToRelease { get; set; } } ``` @@ -421,6 +425,8 @@ public class MigrationFileResult public string FileName { get; set; } = string.Empty; public string ReleaseVersion { get; set; } = string.Empty; public string TargetGroup { get; set; } = string.Empty; + public string? TargetAlias { get; set; } // null when the entry summarizes all targets (a file that succeeded in FileByFile order) + public int MigrationRecordId { get; set; } // 0 when no record exists or applies public bool Success { get; set; } public string? ErrorMessage { get; set; } public DateTime ExecutedAt { get; set; } @@ -453,7 +459,6 @@ public class HashValidationIssue public class MigrationService : IMigrationService { private readonly ILogger _logger; - private readonly IOptions _options; private readonly TemplateExecutor _templateExecutor; private readonly IMigrationContextAccessor _ctxAccessor; private readonly IServiceProvider _serviceProvider; @@ -461,14 +466,12 @@ public class MigrationService : IMigrationService public MigrationService( ILogger logger, - IOptions options, TemplateExecutor templateExecutor, IMigrationContextAccessor ctxAccessor, IServiceProvider serviceProvider, ICliToolExecutor cliToolExecutor) { _logger = logger; - _options = options; _templateExecutor = templateExecutor; _ctxAccessor = ctxAccessor; _serviceProvider = serviceProvider; @@ -477,6 +480,8 @@ public class MigrationService : IMigrationService } ``` +The service reads the configuration of the current run (products, CLI tools, repository) from `_ctxAccessor.Current.RayMigratorOptions`, never from `IOptions`. A host that runs one context per operation therefore needs no options registration in its container; the CLI pipeline binds `IOptions` only to validate the files and hands the value to the context factory. + ### Constants | Constant | Type | Value | Description | @@ -1378,6 +1383,20 @@ public async Task MigrateUp_ShouldSucceed_WhenValidRequest() } ``` +## Progress events + +The three run operations report their progress through `MigrationContext.ProgressObserver` (`IMigrationProgressObserver`, namespace `Raycoon.RayMigrator.Core.Progress`). The public methods are thin wrappers around the execution bodies: they reset the run-started flag and raise the completion from the result, so an operation that ends before its run row exists still reports a matching start. + +| Event | Raised | Carries | +|-------|--------|---------| +| `RunStartedEvent` | once, after the run row exists (migrate-up, migrate-down, baseline); before the completion when the operation ended earlier | product, environment, command, run mode, `MigrationRunId` (0 without a run row), `FromRelease`, `TargetRelease` | +| `FileStartedEvent` | per file and target, after the record exists and the resume offset is known; for rollbacks after the record is set to Executing; for baseline before the record is written | operation (`MigrateUp`, `MigrateDown`, `Rollback` for error recovery, `Baseline`), release, target group, target, file, `FileOrderId`, `MigrationRecordId`, `BlocksTotal`, `ResumeFromBlock` | +| `BlockCompletedEvent` | per executed SQL block on the block-by-block, atomic and simulate paths, migrations and rollbacks; not for files that run through a CLI tool | block number and total, `Success` (false only under the Ignore error action), error message | +| `FileCompletedEvent` | per file and target | `FileOutcome`: `Migrated`, `Recovered`, `Simulated`, `Failed`, `FailedIgnored`, `RolledBack`, `RollbackFailed`, `Baselined`; duration; error message | +| `RunCompletedEvent` | once per operation, as the last event | success, persisted `MigrationRunResult` (null for baseline), the counts of the result, duration, error message | + +On the atomic shared-connection path the blocks of a file commit together with the repository update at the end of the file, so a block reported there is durable only once its `FileCompletedEvent` follows. Validate-hash, update-hash, fix, info and history raise no events. The engine calls the observer synchronously on the thread of the run; an observer that throws is logged at Warning and the run continues. `NullMigrationProgressObserver.Instance` is the default; the CLI sets no observer. + ## DI Registration **Location**: `Raycoon.RayMigrator.Services/ServiceCollectionExtensions.cs` @@ -1395,7 +1414,7 @@ public static IServiceCollection AddRayMigratorServices( if (hostMode == RayMigratorHostMode.Cli) services.AddSingleton(); else - services.AddScoped(); + services.AddSingleton(); services.AddSingleton(); @@ -1403,10 +1422,11 @@ public static IServiceCollection AddRayMigratorServices( } ``` -- **CLI mode** (`RayMigratorHostMode.Cli`): Uses `SingletonMigrationContextAccessor` (single shared context). -- **API mode** (`RayMigratorHostMode.Api`): Uses `AsyncLocalMigrationContextAccessor` (per-request context isolation via `AsyncLocal`). -- `IMigrationContextFactory` is always a singleton (stateless factory). +- **CLI mode** (`RayMigratorHostMode.Cli`): `SingletonMigrationContextAccessor`, one context for the whole process. +- **API mode** (`RayMigratorHostMode.Api`): `AsyncLocalMigrationContextAccessor`, one context per async flow through a static `AsyncLocal`. It is a singleton too; a scoped registration would isolate nothing and would make the singleton factory a captive dependency. +- `IMigrationContextFactory` is a singleton that takes the accessor and a logger; it is the entry point of a host for one run (see [MigrationContext](../02-core-concepts/migration-context.md#hosting-the-engine-in-process)). - `ICliToolExecutor` / `CliToolExecutor` is registered as scoped for external SQL tool execution. +- Not registered here and supplied by the host: `TemplateCache` (one per process), `TemplateExecutor` (scoped per run), `DatabaseLogWriter`, logging. ## Related Documentation diff --git a/Docs/04-service-layer/template-executor.md b/Docs/04-service-layer/template-executor.md index 3568d74..586109c 100644 --- a/Docs/04-service-layer/template-executor.md +++ b/Docs/04-service-layer/template-executor.md @@ -266,16 +266,12 @@ Additionally, `RepositoryMigrationRunInsert` catches `TemplateResultException` w > **Note**: The file lives in the Infrastructure project but uses the `Raycoon.RayMigrator.Core.Templates` namespace. -The `TemplateCache` is eagerly initialized at construction time (not lazy-loaded). It loads all SQL templates from the `DataAccessLayers/{Type}/` filesystem directories, replaces `{ENV:*}` placeholders, and validates that all required templates exist for each configured database type. +The `TemplateCache` is eagerly initialized at construction time (not lazy-loaded). It loads all SQL templates from the `DataAccessLayers/{Type}/` filesystem directories, replaces `{ENV:*}` placeholders, and checks that every template type exists for each loaded database type. It does not depend on a configuration: a host builds one cache per process and calls `ValidateConfigurationAgainstTemplateCache(options)` for each configuration it runs, which checks that the repository and every target group name a database type with templates. The CLI pipeline does that right after building the cache. **Constructor**: ```csharp -public TemplateCache( - IOptions? options, - bool revealSensitiveData, - ILogger logger, - bool validateConfiguration = true) +public TemplateCache(bool revealSensitiveData, ILogger logger) ``` - `options` is nullable. When null (or when `validateConfiguration` is false), configuration validation is deferred. This supports ManagedLocal mode where Products/Repository config is not yet known at startup. diff --git a/Docs/11-development/coding-conventions.md b/Docs/11-development/coding-conventions.md index d816c05..4c63fa4 100644 --- a/Docs/11-development/coding-conventions.md +++ b/Docs/11-development/coding-conventions.md @@ -155,8 +155,8 @@ Reference files are cited so that a rule can be checked against real code. ## 12. Dependency injection - Engine services are registered in `Services/ServiceCollectionExtensions.AddRayMigratorServices` and in `Pipeline/DirectModePipeline` (options, post-configure, validators, singletons). Add new registrations next to their peers; do not create a second composition root. -- Lifetimes: stateless services `Singleton`, per-run services `Scoped` (`IMigrationService`, `RayMigratorService`), `Transient` only for `IPostConfigureOptions`. In CLI mode the context accessor is a singleton; in API mode it is scoped (`AsyncLocal`). -- Options are consumed through `IOptions`; `IOptionsMonitor` and `IOptionsSnapshot` are not used. +- Lifetimes: stateless services `Singleton`, per-run services `Scoped` (`IMigrationService`, `RayMigratorService`, `TemplateExecutor`), `Transient` only for `IPostConfigureOptions`. The context accessor is a singleton in both host modes; the API accessor isolates per async flow through its `AsyncLocal`. +- Services read the configuration of the current run from `MigrationContext.RayMigratorOptions`. `IOptions` is bound only in the CLI composition root to validate the files; `IOptionsMonitor` and `IOptionsSnapshot` are not used. A host creates its context through `IMigrationContextFactory`, never with `new MigrationContext(...)`. - DAL plugins are not in the container. They are discovered by `DalFactory` through `[DatabaseType("SqlServer")]`. - The wizard registers its services as `Scoped` in `ConfigWizard.Web/Program.cs`. diff --git a/Docs/arc42/09-Architecture-Decisions.md b/Docs/arc42/09-Architecture-Decisions.md index 7f0f17f..1176b2e 100644 --- a/Docs/arc42/09-Architecture-Decisions.md +++ b/Docs/arc42/09-Architecture-Decisions.md @@ -33,6 +33,7 @@ followed the same day by the commit tagged `v0.10.3`; a decision already present | ADR-020 | Error handling strategies with rollback files and run results | Accepted (refined 0.13.0, 0.14.0) | <= 0.10.3 | [Rollback Strategies](08-Crosscutting-Concepts.md#rollback-strategies) | | ADR-021 | Configuration arrays merge by alias in every application | Accepted (0.15.0, #23) | 2026-09-23 | [Configuration Inheritance with Validation](08-Crosscutting-Concepts.md#configuration-inheritance-with-validation) | | ADR-022 | Transient error codes ship as an editable file per DAL folder | Accepted (0.16.0, #24) | 2026-10-10 | [Retry on Transient Errors](08-Crosscutting-Concepts.md#retry-on-transient-errors), [DAL Abstraction and plugin discovery](08-Crosscutting-Concepts.md#dal-abstraction-and-plugin-discovery) | +| ADR-023 | The context factory is the hosting seam: one prepared, activated context per run, and typed progress events | Accepted (unreleased) | 2026-10-12 | [Dependency injection and the context pattern](08-Crosscutting-Concepts.md#dependency-injection-and-the-context-pattern) | ## ADRs @@ -466,3 +467,15 @@ the file with the built-in list (rejected, because a built-in code could then ne - [Solution Strategy](04-Solution-Strategy.md), [Crosscutting Concepts](08-Crosscutting-Concepts.md), [Risks and Technical Debt](11-Risks-and-Technical-Debt.md) - [Design decisions](https://github.com/RAYCOON/RayMigrator/blob/main/Docs/01-architecture/design-decisions.md), [Architectural patterns](https://github.com/RAYCOON/RayMigrator/blob/main/Docs/01-architecture/patterns.md), [Changelog](https://github.com/RAYCOON/RayMigrator/blob/main/CHANGELOG.md), [License change dates](https://github.com/RAYCOON/RayMigrator/blob/main/Docs/license-change-dates.md) + +### ADR-023: The context factory is the hosting seam: one prepared, activated context per run, and typed progress events + +**Context.** RayMigrator Studio hosts the engine in one long-running process and runs one migration per request with options it builds in memory. Until 0.16.0 the engine was hostable only on paper: `MigrationService` read the products and the CLI tools from `IOptions` while the repository settings came from the `MigrationContext`, so a context built from other options failed with "Sequence contains no matching element"; `IMigrationContextFactory` existed but no production code used it, and the pipeline and the engine test host repeated the post-build steps (DAL properties, schema names, logging context) by hand; pre-built options skipped the defaults merge, the data annotations and the rule catalog, so null timeouts failed later with an `InvalidOperationException`; and the only progress signal was log text without event ids. + +**Decision.** `IMigrationContextFactory` (implemented in `Services`, registered by `AddRayMigratorServices`) is the entry point of every host for one run. `Create` prepares in-memory options with `RayMigratorOptionsPreparer` (placeholders, defaults, enum probe, data annotations, rule catalog) and `CreateFromValidatedOptions` skips the preparation for the CLI; both fill the DAL properties, check the schema names, register the secrets with the masker and make the context current for the calling async flow. `MigrationService` reads everything from the context; `TemplateCache` takes no options and is validated per configuration. The run operations raise typed events through `MigrationContext.ProgressObserver` (`IMigrationProgressObserver`: run, file on target, block, with ids and outcomes), and refuse a request whose run mode or target release differs from the context. The result DTOs name the repository run (`MigrationRunId`) and the target; history and status are per environment. The accessor is a singleton in both host modes. + +**Consequences.** Positive: a host runs the engine with one factory call per run and no options registration; the CLI and a host take the same path, so the engine test host lost its duplicated steps; progress is observable without parsing log text; a run row records the release it started from. Negative: breaking changes to the service constructor, the factory signature, `TemplateCache`, the result DTOs and the `Repository_MigrationRun_Select` template (allowed by ADR-016); `DirectModePipeline` remains CLI-shaped (static `Log.Logger`, console output, exit code) and is not the host path; the observer is called synchronously on the thread of the run, so a slow observer slows the run. + +**Alternatives considered.** Keeping `IOptions` and registering one container per configuration in the host (rejected: every run would build a container, and the two option sources could still diverge). Reporting progress through a Serilog sink only (rejected: the per-file message has no event id and changes with the wording). Raising the run-started event before the run row exists (rejected: a host needs the run id to correlate the live run with the repository). + +**References.** `Raycoon.RayMigrator.Core/IMigrationContextFactory.cs`, `Raycoon.RayMigrator.Services/MigrationContextFactory.cs`, `Raycoon.RayMigrator.Core/Configuration/Validation/RayMigratorOptionsPreparer.cs`, `Raycoon.RayMigrator.Core/Progress/`, [MigrationContext: Hosting the engine in-process](../02-core-concepts/migration-context.md#hosting-the-engine-in-process), `CHANGELOG.md` (Unreleased). From c95f0cda787524868cb2fe9f2e6e07b6018c3224 Mon Sep 17 00:00:00 2001 From: danielpavic Date: Mon, 12 Oct 2026 22:30:23 +0200 Subject: [PATCH 07/14] Sync the remaining docs and the example template with the result changes The architecture pattern and component tables still showed the Guid RunId and the InitiatedBy placeholder, and the external-DAL table and the example plugin's template skeleton described Repository_MigrationRun_Select as a query by product; it takes @EnvironmentId as well now. --- Docs/01-architecture/component-responsibilities.md | 12 ++++++------ Docs/01-architecture/patterns.md | 2 +- Docs/09-extending/external-dal-development.md | 2 +- .../Templates/Repository_MigrationRun_Select.sql | 6 ++++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Docs/01-architecture/component-responsibilities.md b/Docs/01-architecture/component-responsibilities.md index 7ae77b0..f16880d 100644 --- a/Docs/01-architecture/component-responsibilities.md +++ b/Docs/01-architecture/component-responsibilities.md @@ -151,16 +151,16 @@ All request types are in `Models/Requests.cs`, all result types are in `Models/R | Result DTO | Purpose | |-----|---------| | `OperationResult` | Abstract base class (Success, ErrorMessage, ErrorCode, Messages, ExecutedAt, Duration). `ErrorCode` is `int?`: negative = SQL template ResultCode, positive = C# backend ErrorCode, null = unclassified. | -| `MigrationOperationResult` | Result of migration up/down (extends `OperationResult`). Adds `RunId` (Guid), `ProductAlias`, `Environment`, `Operation`, `Result`, `TotalMigrations`, `SuccessfulMigrations`, `FailedMigrations`, `CurrentRelease`, `MigrationResults` (List). | -| `MigrationFileResult` | Result of a single migration file execution (FileName, ReleaseVersion, TargetGroup, Success, ErrorMessage, ExecutedAt, Duration). | +| `MigrationOperationResult` | Result of migration up/down (extends `OperationResult`). Adds `MigrationRunId` (the repository run row, 0 without one), `ProductAlias`, `Environment`, `Operation`, `Result`, `TotalMigrations`, `SuccessfulMigrations`, `FailedMigrations`, `CurrentRelease`, `MigrationResults` (List). | +| `MigrationFileResult` | Result of a single migration file execution (FileName, ReleaseVersion, TargetGroup, TargetAlias (null when the entry summarizes all targets of the group), MigrationRecordId, Success, ErrorMessage, ExecutedAt, Duration). | | `ValidationResult` | Result of hash validation (TotalFiles, ValidFiles, InvalidFiles, MissingFiles, Issues). | | `HashValidationIssue` | Individual hash validation issue (FileName, IssueType, ExpectedHash, ActualHash, Details). | | `HashUpdateResult` | Result of hash update (UpdatedFiles, NewFiles, RemovedFiles, UpdatedFileNames). | -| `BaselineResult` | Result of baseline operation (TargetReleaseVersion, BaselinedFiles). | -| `MigrationStatusInfo` | Current migration status for a product (CurrentRelease, LastMigrationDate, TotalMigrationsExecuted, PendingMigrations, LastRunResult, TargetGroups). | +| `BaselineResult` | Result of baseline operation (Environment, MigrationRunId, TargetReleaseVersion, BaselinedFiles). | +| `MigrationStatusInfo` | Current migration status for a product and environment (Environment, CurrentRelease, LastMigrationDate, TotalMigrationsExecuted, PendingMigrations, LastRunResult, TargetGroups). | | `TargetGroupStatus` | Status of a target group (Alias, DatabaseType, CurrentRelease, ExecutedMigrations, LastMigrationDate, Targets). | -| `MigrationHistory` | Migration run history (Runs: List). | -| `MigrationRunInfo` | Information about a migration run (MigrationRunId, RunId, StartedAt, CompletedAt, Operation, Result, RunMode, InitiatedBy, TotalMigrations, SuccessfulMigrations, FailedMigrations, ToRelease). | +| `MigrationHistory` | Migration run history of a product and environment (Environment, Runs: List). | +| `MigrationRunInfo` | Information about a migration run (MigrationRunId, Environment, StartedAt, CompletedAt, Operation, Result, RunMode, TotalMigrations, SuccessfulMigrations, FailedMigrations, FromRelease, ToRelease). | | `FixIssuesResult` | Result of Fix command (WasSimulated, OrphanedRunsFound, OrphanedRunsFixed, OrphanedRuns: List). | | `OrphanedRunInfo` | Information about an orphaned MigrationRun entry (MigrationRunId, Environment, EnvironmentId, StartedAt, MinutesRunning, MigrationRunModeId, WasFixed). | diff --git a/Docs/01-architecture/patterns.md b/Docs/01-architecture/patterns.md index 32d416e..f96e06d 100644 --- a/Docs/01-architecture/patterns.md +++ b/Docs/01-architecture/patterns.md @@ -515,7 +515,7 @@ public abstract class OperationResult public class MigrationOperationResult : OperationResult { - public Guid RunId { get; set; } + public int MigrationRunId { get; set; } // MigrationRun.Id; 0 when the run mode writes no run row public string ProductAlias { get; set; } = string.Empty; public string Environment { get; set; } = string.Empty; public MigrationOperation Operation { get; set; } diff --git a/Docs/09-extending/external-dal-development.md b/Docs/09-extending/external-dal-development.md index ea244bc..34f44c2 100644 --- a/Docs/09-extending/external-dal-development.md +++ b/Docs/09-extending/external-dal-development.md @@ -248,7 +248,7 @@ The `Database.Example` project includes all 22 files as placeholders with TODO c | `Repository_MigrationRecord_UpdateRollback.sql` | Update after rollback (includes inline MigrationRecordHistory insert) | | `Repository_MigrationRun_FixOrphaned.sql` | Fix orphaned run records | | `Repository_MigrationRun_Insert.sql` | Insert migration run record | -| `Repository_MigrationRun_Select.sql` | Select current run for product | +| `Repository_MigrationRun_Select.sql` | Select the newest runs of a product and environment (`@ProductId`, `@EnvironmentId`, `@Limit`) | | `Repository_MigrationRun_SelectOrphaned.sql` | Select orphaned runs | | `Repository_MigrationRun_Update.sql` | Update run result | | `Repository_Product_CheckInsert.sql` | Lookup product by `NameLower`, insert if not exists; returns `ProductId` | diff --git a/Raycoon.RayMigrator.Database.Example/Templates/Repository_MigrationRun_Select.sql b/Raycoon.RayMigrator.Database.Example/Templates/Repository_MigrationRun_Select.sql index a627860..a856ce8 100644 --- a/Raycoon.RayMigrator.Database.Example/Templates/Repository_MigrationRun_Select.sql +++ b/Raycoon.RayMigrator.Database.Example/Templates/Repository_MigrationRun_Select.sql @@ -1,8 +1,10 @@ -- TODO: Implement for your database engine --- Purpose: Selects the current migration run for a product +-- Purpose: Selects the newest migration runs of a product and environment (history and status) -- See the corresponding template in Database.SqlServer or Database.PostgreSQL for reference. -- -- Parameters used in this template: -- {CFG:SchemaName} - The repository schema name -- {CFG:TableBaseName} - The table name prefix --- @ParameterName - SQL parameters (see reference implementations) +-- @ProductId - The product to query +-- @EnvironmentId - The environment to query; runs of other environments are not returned +-- @Limit - Maximum number of runs, newest first From 46552439e9a94fd7c1da10aa62a2275c1abc1be2 Mon Sep 17 00:00:00 2001 From: danielpavic Date: Mon, 12 Oct 2026 23:56:05 +0200 Subject: [PATCH 08/14] Pair every progress event and report the pair a failure hits The migrate-up catch blocks read the target and the record id from MigrationState, which is assigned after the record insert, so a failure before the insert was attributed to the previous file and target, and the Failed update went to the previous record. The loops now keep the pair they work on in locals; a failure before the insert marks no record. Every FileCompletedEvent now follows a FileStartedEvent for the same file and target: a record a previous run completed but never finalized reports a start before Recovered; a rollback reports its start once per record, so the paths that end before the rollback file is parsed (missing file, lookup failure, the outer catch) pair as well, and a record without a rollback file reports the new outcome Skipped instead of nothing; a baseline reports the start after the record exists, and a failure completes the file it hit. The rollback start carries the resume offset, the start of a CLI-tool file one block, and a file-by-file run stamps the start time per target. The atomic paths raise their block events after the commit, so a transient retry of the batch repeats no event. Simulate and Validate set FromReleaseVersion from the records they read, so RunStartedEvent reports where the product stands without a run row. BaselineResult.BaselinedRecords counts the records written, one per file and target, also when the baseline failed midway; the baseline run events use it. BaselinedFiles keeps its meaning. --- CHANGELOG.md | 12 +- Docs/04-service-layer/migration-service.md | 10 +- .../Progress/FileOutcome.cs | 3 + .../Progress/FileStartedEvent.cs | 3 +- .../Progress/NullMigrationProgressObserver.cs | 6 + .../Models/Results.cs | 4 + .../MigrationService.cs | 189 ++++++++++++------ .../Features/SqliteProgressEventsTests.cs | 61 +++++- 8 files changed, 220 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 579c50a..5d8ad07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,9 +15,15 @@ RayMigrator follows Semantic Versioning where applicable. `FileStartedEvent` and `FileCompletedEvent` for every file on every target (migration, rollback and baseline, with the record id and a `FileOutcome`), `BlockCompletedEvent` for every executed SQL block, and `RunCompletedEvent` - with the result. A host shows progress without parsing log text. An - observer that throws is logged and the run continues. The default observer - ignores every event; the CLI does not use them. + with the result. Every completion follows a start for the same file and + target; a rollback without a rollback file reports `Skipped`; the block + events of the atomic path follow the commit; and `FromRelease` is reported + in Simulate and Validate as well. A host shows progress without parsing + log text. An observer that throws is logged and the run continues. The + default observer ignores every event; the CLI does not use them. +- `BaselineResult.BaselinedRecords` counts the records a baseline wrote, one + per file and target that was pending, also when the baseline failed midway. + `BaselinedFiles` keeps counting files per release and target group. ### Changed diff --git a/Docs/04-service-layer/migration-service.md b/Docs/04-service-layer/migration-service.md index 4a98208..443d671 100644 --- a/Docs/04-service-layer/migration-service.md +++ b/Docs/04-service-layer/migration-service.md @@ -1389,13 +1389,13 @@ The three run operations report their progress through `MigrationContext.Progres | Event | Raised | Carries | |-------|--------|---------| -| `RunStartedEvent` | once, after the run row exists (migrate-up, migrate-down, baseline); before the completion when the operation ended earlier | product, environment, command, run mode, `MigrationRunId` (0 without a run row), `FromRelease`, `TargetRelease` | -| `FileStartedEvent` | per file and target, after the record exists and the resume offset is known; for rollbacks after the record is set to Executing; for baseline before the record is written | operation (`MigrateUp`, `MigrateDown`, `Rollback` for error recovery, `Baseline`), release, target group, target, file, `FileOrderId`, `MigrationRecordId`, `BlocksTotal`, `ResumeFromBlock` | -| `BlockCompletedEvent` | per executed SQL block on the block-by-block, atomic and simulate paths, migrations and rollbacks; not for files that run through a CLI tool | block number and total, `Success` (false only under the Ignore error action), error message | -| `FileCompletedEvent` | per file and target | `FileOutcome`: `Migrated`, `Recovered`, `Simulated`, `Failed`, `FailedIgnored`, `RolledBack`, `RollbackFailed`, `Baselined`; duration; error message | +| `RunStartedEvent` | once, after the run row exists (migrate-up, migrate-down, baseline) or, in a run mode without a run row, after the repository records were read; before the completion when the operation ended earlier | product, environment, command, run mode, `MigrationRunId` (0 without a run row), `FromRelease` (the highest migrated release, in Simulate and Validate as well), `TargetRelease` | +| `FileStartedEvent` | per file and target, after the record exists and the resume offset and the executor (DAL or CLI tool) are known; a record a previous run completed but never finalized is reported as started and `Recovered` without SQL; for rollbacks once the record is taken up, before its rollback file is looked up; for baseline after the record exists | operation (`MigrateUp`, `MigrateDown`, `Rollback` for error recovery, `Baseline`), release, target group, target, file, `FileOrderId`, `MigrationRecordId`, `BlocksTotal` (1 for a CLI tool; for a rollback that never reached its file, what the repository recorded), `ResumeFromBlock` | +| `BlockCompletedEvent` | per executed SQL block: on the block-by-block and simulate paths as the block completes, on the atomic path after the commit of the file; migrations and rollbacks; not for files that run through a CLI tool | block number and total, `Success` (false only under the Ignore error action), error message | +| `FileCompletedEvent` | per file and target, always after its `FileStartedEvent`; a failure before the record insert names the file and target it hit, with record id 0 | `FileOutcome`: `Migrated`, `Recovered`, `Simulated`, `Failed`, `FailedIgnored`, `RolledBack`, `RollbackFailed`, `Skipped` (a rollback without a rollback file, or a chain that stopped at one), `Baselined`; duration; error message | | `RunCompletedEvent` | once per operation, as the last event | success, persisted `MigrationRunResult` (null for baseline), the counts of the result, duration, error message | -On the atomic shared-connection path the blocks of a file commit together with the repository update at the end of the file, so a block reported there is durable only once its `FileCompletedEvent` follows. Validate-hash, update-hash, fix, info and history raise no events. The engine calls the observer synchronously on the thread of the run; an observer that throws is logged at Warning and the run continues. `NullMigrationProgressObserver.Instance` is the default; the CLI sets no observer. +On the atomic shared-connection path the blocks of a file commit together with the repository update at the end of the file; their events are raised after the commit, so a reported block is durable and a transient retry of the batch repeats no event. Validate-hash, update-hash, fix, info and history raise no events. The engine calls the observer synchronously on the thread of the run; an observer that throws is logged at Warning and the run continues. `NullMigrationProgressObserver.Instance` is the default; the CLI sets no observer. ## DI Registration diff --git a/Raycoon.RayMigrator.Core/Progress/FileOutcome.cs b/Raycoon.RayMigrator.Core/Progress/FileOutcome.cs index 1f6494c..2d55c29 100644 --- a/Raycoon.RayMigrator.Core/Progress/FileOutcome.cs +++ b/Raycoon.RayMigrator.Core/Progress/FileOutcome.cs @@ -30,4 +30,7 @@ public enum FileOutcome /// Baseline recorded the file as Migrated without executing it. Baselined = 80, + + /// Nothing ran on the target: a rollback without a rollback file, or a rollback chain that stopped at one. + Skipped = 90, } diff --git a/Raycoon.RayMigrator.Core/Progress/FileStartedEvent.cs b/Raycoon.RayMigrator.Core/Progress/FileStartedEvent.cs index b977953..f190542 100644 --- a/Raycoon.RayMigrator.Core/Progress/FileStartedEvent.cs +++ b/Raycoon.RayMigrator.Core/Progress/FileStartedEvent.cs @@ -12,7 +12,8 @@ namespace Raycoon.RayMigrator.Core.Progress; /// The file name without path. /// The position of the file in the execution order. /// The repository record of the file on the target, or 0 when the run mode writes no records. -/// The number of SQL blocks of the file; 1 for a file that runs through a CLI tool. +/// The number of SQL blocks of the file; 1 for a file that runs through a CLI tool. For a rollback that +/// ends before its rollback file is parsed, the count the repository recorded for the record. /// The number of leading blocks a previous run already committed and this run skips. /// When the file started. public sealed record FileStartedEvent( diff --git a/Raycoon.RayMigrator.Core/Progress/NullMigrationProgressObserver.cs b/Raycoon.RayMigrator.Core/Progress/NullMigrationProgressObserver.cs index d369f7c..10e591a 100644 --- a/Raycoon.RayMigrator.Core/Progress/NullMigrationProgressObserver.cs +++ b/Raycoon.RayMigrator.Core/Progress/NullMigrationProgressObserver.cs @@ -5,28 +5,34 @@ namespace Raycoon.RayMigrator.Core.Progress; /// public sealed class NullMigrationProgressObserver : IMigrationProgressObserver { + /// The single instance. public static NullMigrationProgressObserver Instance { get; } = new(); private NullMigrationProgressObserver() { } + /// public void OnRunStarted(RunStartedEvent e) { } + /// public void OnFileStarted(FileStartedEvent e) { } + /// public void OnBlockCompleted(BlockCompletedEvent e) { } + /// public void OnFileCompleted(FileCompletedEvent e) { } + /// public void OnRunCompleted(RunCompletedEvent e) { } diff --git a/Raycoon.RayMigrator.Services.Abstractions/Models/Results.cs b/Raycoon.RayMigrator.Services.Abstractions/Models/Results.cs index 634f8bd..13a2eab 100644 --- a/Raycoon.RayMigrator.Services.Abstractions/Models/Results.cs +++ b/Raycoon.RayMigrator.Services.Abstractions/Models/Results.cs @@ -118,7 +118,11 @@ public class BaselineResult : OperationResult /// The repository id of the run row (MigrationRun.Id), or 0 when the run failed before it was inserted. public int MigrationRunId { get; set; } public string? TargetReleaseVersion { get; set; } + /// Files recorded per release and target group, whether or not every target still needed them. public int BaselinedFiles { get; set; } + + /// Records written: one per file and target that was pending. Also set when the baseline failed midway. + public int BaselinedRecords { get; set; } } /// diff --git a/Raycoon.RayMigrator.Services/MigrationService.cs b/Raycoon.RayMigrator.Services/MigrationService.cs index 0519307..18c401d 100644 --- a/Raycoon.RayMigrator.Services/MigrationService.cs +++ b/Raycoon.RayMigrator.Services/MigrationService.cs @@ -122,8 +122,6 @@ private async Task MigrateUpCoreAsync(MigrateUpRequest await RepositoryMigrationRunInsertWithAutoFix(settingsJson); } - ReportRunStarted(MigrationCommand.MigrateUp, request.RunMode, request.TargetReleaseVersion); - // --- Phase 2: File Discovery & Preparation --- var productOptions = _ctxAccessor.Current.RayMigratorOptions.Products!.First(p => p.Alias == request.ProductAlias); var migrationFiles = DiscoverAndPrepareMigrationFiles(productOptions, request.Environment); @@ -186,6 +184,14 @@ private async Task MigrateUpCoreAsync(MigrateUpRequest filesToMigrate = migrationFiles; // Validate: process all files } + // A run mode that writes no run row has no FromReleaseVersion yet; it comes from the records read above + if (!request.RunMode.ShouldWriteRepository()) + { + _ctxAccessor.Current.MigrationState.FromReleaseVersion = GetHighestMigratedRelease(existingRecords); + } + + ReportRunStarted(MigrationCommand.MigrateUp, request.RunMode, request.TargetReleaseVersion); + // Filter by target release if specified filesToMigrate = FilterByTargetRelease(filesToMigrate, request.TargetReleaseVersion); @@ -468,6 +474,12 @@ internal async Task ExecuteTargetGroupFileByFile( var errorAction = file.MigrationErrorActionOverride ?? productOptions.MigrationErrorActionEnum; bool ignoreErrors = errorAction == MigrationErrorAction.Ignore; + // The target and record the loop below is working on. The catch reads them instead of MigrationState, + // which is assigned after the record insert and would still name the previous pair before it. + string currentTargetAlias = string.Empty; + int currentRecordId = 0; + var targetStartTime = fileStartTime; + try { bool fileHadBlockFailures = false; @@ -481,6 +493,10 @@ internal async Task ExecuteTargetGroupFileByFile( continue; } + currentTargetAlias = targetOptions.Alias!; + currentRecordId = 0; + targetStartTime = DateTime.UtcNow; + // OPT-1: Check if a previous run completed all blocks but wasn't finalized if (request.RunMode.ShouldWriteRepository()) { @@ -488,7 +504,8 @@ internal async Task ExecuteTargetGroupFileByFile( if (finalizedId > 0) { successfullyMigratedRecords.Add((file, finalizedId, targetOptions.Alias!)); - ReportFileCompleted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, finalizedId, FileOutcome.Recovered, fileStartTime); + ReportFileStarted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, finalizedId, file.FileUpBlocksTotal, targetStartTime); + ReportFileCompleted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, finalizedId, FileOutcome.Recovered, targetStartTime); _logger.LogInformation( "Migration recovered | Product: {Product} | Env: {Environment} | Release: {Release} | TargetGroup: {TargetGroup} | Target: {Target} | File: {Filename} | MigrationRecordId: {MigrationRecordId} (finalized from previous incomplete run)", productOptions.Alias, _ctxAccessor.Current.RayMigratorConsoleOptions.Environment, file.ReleaseVersion, targetGroupOptions.Alias, targetOptions.Alias, file.Filename, finalizedId); @@ -525,6 +542,7 @@ internal async Task ExecuteTargetGroupFileByFile( migrationRecordId = existingMigrationRecordId; } + currentRecordId = migrationRecordId; _ctxAccessor.Current.MigrationState.MigrationRecordId = migrationRecordId; _ctxAccessor.Current.MigrationState.ReleaseVersionFromFileNameWithPath = file.ReleaseVersion; _ctxAccessor.Current.MigrationState.FilenameWithRelativePath = file.FilenameWithRelativePath; @@ -537,12 +555,12 @@ internal async Task ExecuteTargetGroupFileByFile( // Check if this file+target can resume from a previous partial execution int startFromBlock = FindResumableBlock(file, targetOptions.Alias!, existingRecords); - ReportFileStarted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, migrationRecordId, startFromBlock, fileStartTime); - // Branch: CLI tool or DAL execution int succeededBlocks, failedBlocks; bool atomicCommitCompleted; string? resolvedCliAlias = ResolveUseCliToolAlias(file, targetOptions); + ReportFileStarted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, migrationRecordId, startFromBlock, targetStartTime, + blocksTotal: resolvedCliAlias != null ? 1 : file.FileUpBlocksTotal); if (resolvedCliAlias != null) { var cliTool = GetCliToolByAlias(resolvedCliAlias); @@ -560,7 +578,7 @@ internal async Task ExecuteTargetGroupFileByFile( if (failedBlocks > 0) { - ReportFileCompleted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, migrationRecordId, FileOutcome.FailedIgnored, fileStartTime, + ReportFileCompleted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, migrationRecordId, FileOutcome.FailedIgnored, targetStartTime, $"{failedBlocks}/{file.FileUpBlocksTotal} block(s) failed"); // Block-level failures with Ignore: mark as Failed, skip remaining targets for this file fileHadBlockFailures = true; @@ -589,7 +607,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( successfullyMigratedRecords.Add((file, migrationRecordId, targetOptions.Alias!)); ReportFileCompleted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, migrationRecordId, - request.RunMode.ShouldExecuteSql() ? FileOutcome.Migrated : FileOutcome.Simulated, fileStartTime); + request.RunMode.ShouldExecuteSql() ? FileOutcome.Migrated : FileOutcome.Simulated, targetStartTime); _logger.LogInformation( "Migration successful | Product: {Product} | Env: {Environment} | Release: {Release} | TargetGroup: {TargetGroup} | Target: {Target} | File: {Filename} | MigrationRecordId: {MigrationRecordId} | SqlBlocks: {SqlBlocksTotal}", @@ -625,18 +643,19 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( } catch (Exception ex) { - ReportFileCompleted(MigrationOperation.MigrateUp, file, _ctxAccessor.Current.MigrationState.TargetAlias, - _ctxAccessor.Current.MigrationState.MigrationRecordId, ignoreErrors ? FileOutcome.FailedIgnored : FileOutcome.Failed, fileStartTime, ex.Message); + ReportFileCompleted(MigrationOperation.MigrateUp, file, currentTargetAlias, currentRecordId, + ignoreErrors ? FileOutcome.FailedIgnored : FileOutcome.Failed, targetStartTime, ex.Message); _logger.LogError(ex, "Migration FAILED | Product: {Product} | Env: {Environment} | Release: {Release} | TargetGroup: {TargetGroup} | Target: {Target} | File: {Filename} | MigrationRecordId: {MigrationRecordId} | SqlBlock: {SqlBlock}/{SqlBlocksTotal}", - productOptions.Alias, _ctxAccessor.Current.RayMigratorConsoleOptions.Environment, file.ReleaseVersion, targetGroupOptions.Alias, _ctxAccessor.Current.MigrationState.TargetAlias, file.Filename, _ctxAccessor.Current.MigrationState.MigrationRecordId, _ctxAccessor.Current.MigrationState.FileBlockId, file.FileUpBlocksTotal); + productOptions.Alias, _ctxAccessor.Current.RayMigratorConsoleOptions.Environment, file.ReleaseVersion, targetGroupOptions.Alias, currentTargetAlias, file.Filename, currentRecordId, _ctxAccessor.Current.MigrationState.FileBlockId, file.FileUpBlocksTotal); try { - if (request.RunMode.ShouldWriteRepository()) + // A failure before the record insert has no record to mark + if (request.RunMode.ShouldWriteRepository() && currentRecordId > 0) { await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( - _ctxAccessor.Current.MigrationState.MigrationRecordId, MigrationStatus.Failed, + currentRecordId, MigrationStatus.Failed, ignoreErrors ? file.FileUpBlocksTotal : _ctxAccessor.Current.MigrationState.FileBlocksCommitted)); } } @@ -644,7 +663,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( { _logger.LogWarning(updateEx, "Failed to update migration record {MigrationRecordId} ({Filename}) for error state", - _ctxAccessor.Current.MigrationState.MigrationRecordId, file.Filename); + currentRecordId, file.Filename); } if (ignoreErrors) @@ -652,7 +671,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( // Ignore: log warning, mark as Failed, continue to next file _logger.LogWarning( "MigrationErrorAction=Ignore: Exception in {Filename} on target {Target}. Continuing with next file.", - file.Filename, _ctxAccessor.Current.MigrationState.TargetAlias); + file.Filename, currentTargetAlias); result.FailCount++; migrationResults.Add(new MigrationFileResult @@ -660,8 +679,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( FileName = file.Filename, ReleaseVersion = file.ReleaseVersion, TargetGroup = file.TargetGroupAlias, - TargetAlias = _ctxAccessor.Current.MigrationState.TargetAlias, - MigrationRecordId = _ctxAccessor.Current.MigrationState.MigrationRecordId, + TargetAlias = currentTargetAlias, + MigrationRecordId = currentRecordId, Success = false, ErrorMessage = ex.Message, ExecutedAt = fileStartTime, @@ -674,7 +693,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( result.FailCount++; result.Success = false; result.FailedFile = file; - result.FailedMigrationRecordId = _ctxAccessor.Current.MigrationState.MigrationRecordId; + result.FailedMigrationRecordId = currentRecordId; result.ErrorMessage = ex.Message; migrationResults.Add(new MigrationFileResult @@ -682,8 +701,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( FileName = file.Filename, ReleaseVersion = file.ReleaseVersion, TargetGroup = file.TargetGroupAlias, - TargetAlias = _ctxAccessor.Current.MigrationState.TargetAlias, - MigrationRecordId = _ctxAccessor.Current.MigrationState.MigrationRecordId, + TargetAlias = currentTargetAlias, + MigrationRecordId = currentRecordId, Success = false, ErrorMessage = ex.Message, ExecutedAt = fileStartTime, @@ -728,6 +747,9 @@ internal async Task ExecuteTargetGroupTargetByTarget var errorAction = file.MigrationErrorActionOverride ?? productOptions.MigrationErrorActionEnum; bool ignoreErrors = errorAction == MigrationErrorAction.Ignore; + // The record this pair is working on; MigrationState would still name the previous pair before the insert. + int currentRecordId = 0; + try { // OPT-1: Check if a previous run completed all blocks but wasn't finalized @@ -737,6 +759,7 @@ internal async Task ExecuteTargetGroupTargetByTarget if (finalizedId > 0) { successfullyMigratedRecords.Add((file, finalizedId, targetOptions.Alias!)); + ReportFileStarted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, finalizedId, file.FileUpBlocksTotal, fileStartTime); ReportFileCompleted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, finalizedId, FileOutcome.Recovered, fileStartTime); _logger.LogInformation( "Migration recovered | Product: {Product} | Env: {Environment} | Release: {Release} | TargetGroup: {TargetGroup} | Target: {Target} | File: {Filename} | MigrationRecordId: {MigrationRecordId} (finalized from previous incomplete run)", @@ -787,6 +810,7 @@ internal async Task ExecuteTargetGroupTargetByTarget migrationRecordId = existingMigrationRecordId; } + currentRecordId = migrationRecordId; _ctxAccessor.Current.MigrationState.MigrationRecordId = migrationRecordId; _ctxAccessor.Current.MigrationState.ReleaseVersionFromFileNameWithPath = file.ReleaseVersion; _ctxAccessor.Current.MigrationState.FilenameWithRelativePath = file.FilenameWithRelativePath; @@ -799,12 +823,12 @@ internal async Task ExecuteTargetGroupTargetByTarget // Check if this file+target can resume from a previous partial execution int startFromBlock = FindResumableBlock(file, targetOptions.Alias!, existingRecords); - ReportFileStarted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, migrationRecordId, startFromBlock, fileStartTime); - // Branch: CLI tool or DAL execution int succeededBlocks, failedBlocks; bool atomicCommitCompleted; string? resolvedCliAlias = ResolveUseCliToolAlias(file, targetOptions); + ReportFileStarted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, migrationRecordId, startFromBlock, fileStartTime, + blocksTotal: resolvedCliAlias != null ? 1 : file.FileUpBlocksTotal); if (resolvedCliAlias != null) { var cliTool = GetCliToolByAlias(resolvedCliAlias); @@ -883,18 +907,19 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( } catch (Exception ex) { - ReportFileCompleted(MigrationOperation.MigrateUp, file, _ctxAccessor.Current.MigrationState.TargetAlias, - _ctxAccessor.Current.MigrationState.MigrationRecordId, ignoreErrors ? FileOutcome.FailedIgnored : FileOutcome.Failed, fileStartTime, ex.Message); + ReportFileCompleted(MigrationOperation.MigrateUp, file, targetOptions.Alias!, currentRecordId, + ignoreErrors ? FileOutcome.FailedIgnored : FileOutcome.Failed, fileStartTime, ex.Message); _logger.LogError(ex, "Migration FAILED | Product: {Product} | Env: {Environment} | Release: {Release} | TargetGroup: {TargetGroup} | Target: {Target} | File: {Filename} | MigrationRecordId: {MigrationRecordId} | SqlBlock: {SqlBlock}/{SqlBlocksTotal}", - productOptions.Alias, _ctxAccessor.Current.RayMigratorConsoleOptions.Environment, file.ReleaseVersion, targetGroupOptions.Alias, targetOptions.Alias, file.Filename, _ctxAccessor.Current.MigrationState.MigrationRecordId, _ctxAccessor.Current.MigrationState.FileBlockId, file.FileUpBlocksTotal); + productOptions.Alias, _ctxAccessor.Current.RayMigratorConsoleOptions.Environment, file.ReleaseVersion, targetGroupOptions.Alias, targetOptions.Alias, file.Filename, currentRecordId, _ctxAccessor.Current.MigrationState.FileBlockId, file.FileUpBlocksTotal); try { - if (request.RunMode.ShouldWriteRepository()) + // A failure before the record insert has no record to mark + if (request.RunMode.ShouldWriteRepository() && currentRecordId > 0) { await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( - _ctxAccessor.Current.MigrationState.MigrationRecordId, MigrationStatus.Failed, + currentRecordId, MigrationStatus.Failed, ignoreErrors ? file.FileUpBlocksTotal : _ctxAccessor.Current.MigrationState.FileBlocksCommitted)); } } @@ -902,7 +927,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( { _logger.LogWarning(updateEx, "Failed to update migration record {MigrationRecordId} ({Filename}) for error state", - _ctxAccessor.Current.MigrationState.MigrationRecordId, file.Filename); + currentRecordId, file.Filename); } if (ignoreErrors) @@ -919,7 +944,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( ReleaseVersion = file.ReleaseVersion, TargetGroup = file.TargetGroupAlias, TargetAlias = targetOptions.Alias, - MigrationRecordId = _ctxAccessor.Current.MigrationState.MigrationRecordId, + MigrationRecordId = currentRecordId, Success = false, ErrorMessage = ex.Message, ExecutedAt = fileStartTime, @@ -932,7 +957,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( result.FailCount++; result.Success = false; result.FailedFile = file; - result.FailedMigrationRecordId = _ctxAccessor.Current.MigrationState.MigrationRecordId; + result.FailedMigrationRecordId = currentRecordId; result.ErrorMessage = ex.Message; migrationResults.Add(new MigrationFileResult @@ -941,7 +966,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( ReleaseVersion = file.ReleaseVersion, TargetGroup = file.TargetGroupAlias, TargetAlias = targetOptions.Alias, - MigrationRecordId = _ctxAccessor.Current.MigrationState.MigrationRecordId, + MigrationRecordId = currentRecordId, Success = false, ErrorMessage = ex.Message, ExecutedAt = fileStartTime, @@ -1008,15 +1033,17 @@ private void ReportRunCompleted( ctx.MigrationState.MigrationRunId, success, result, totalFiles, succeededFiles, failedFiles, duration, errorMessage))); } - private void ReportFileStarted(MigrationOperation operation, MigrationFileInfo file, string targetAlias, int migrationRecordId, int resumeFromBlock, DateTime startedAtUtc) + private void ReportFileStarted( + MigrationOperation operation, MigrationFileInfo file, string targetAlias, int migrationRecordId, int resumeFromBlock, DateTime startedAtUtc, + int? blocksTotal = null) => ReportFileStarted(operation, file.ReleaseVersion, file.TargetGroupAlias, targetAlias, file.Filename, file.FileOrderId, migrationRecordId, - Math.Max(file.FileUpBlocksTotal, 1), resumeFromBlock, startedAtUtc); + blocksTotal ?? file.FileUpBlocksTotal, resumeFromBlock, startedAtUtc); private void ReportFileStarted( MigrationOperation operation, string releaseVersion, string targetGroupAlias, string targetAlias, string fileName, int fileOrderId, int migrationRecordId, int blocksTotal, int resumeFromBlock, DateTime startedAtUtc) => Report(nameof(IMigrationProgressObserver.OnFileStarted), o => o.OnFileStarted(new FileStartedEvent( - operation, releaseVersion, targetGroupAlias, targetAlias, fileName, fileOrderId, migrationRecordId, blocksTotal, resumeFromBlock, startedAtUtc))); + operation, releaseVersion, targetGroupAlias, targetAlias, fileName, fileOrderId, migrationRecordId, Math.Max(blocksTotal, 1), resumeFromBlock, startedAtUtc))); private void ReportFileCompleted(MigrationOperation operation, MigrationFileInfo file, string targetAlias, int migrationRecordId, FileOutcome outcome, DateTime startedAtUtc, string? errorMessage = null) => ReportFileCompleted(operation, file.ReleaseVersion, file.TargetGroupAlias, targetAlias, file.Filename, migrationRecordId, outcome, startedAtUtc, errorMessage); @@ -1365,8 +1392,6 @@ private async Task MigrateDownCoreAsync(MigrateDownReq await RepositoryMigrationRunInsertWithAutoFix(settingsJson); } - ReportRunStarted(MigrationCommand.MigrateDown, request.RunMode, request.TargetReleaseVersion); - // --- Phase 2: Query migrations for rollback --- // RepositoryMigrationSelect always reads Migrate-mode records (the only ones that exist). List existingRecords; @@ -1441,6 +1466,13 @@ private async Task MigrateDownCoreAsync(MigrateDownReq _logger.LogInformation("Rolling back {Count} migration(s) for product {Product} to release {Release}", migrationsToRollback.Count, request.ProductAlias, request.TargetReleaseVersion); + if (!request.RunMode.ShouldWriteRepository()) + { + _ctxAccessor.Current.MigrationState.FromReleaseVersion = GetHighestMigratedRelease(existingRecords); + } + + ReportRunStarted(MigrationCommand.MigrateDown, request.RunMode, request.TargetReleaseVersion); + // --- Phase 3: Execute rollbacks --- var productOptions = _ctxAccessor.Current.RayMigratorOptions.Products!.First(p => p.Alias == request.ProductAlias); @@ -1543,13 +1575,17 @@ public async Task BaselineAsync(BaselineRequest request) } ReportRunCompleted(MigrationCommand.Baseline, MigrationRunMode.Migrate, request.TargetReleaseVersion, - result.Success, null, result.BaselinedFiles, result.BaselinedFiles, 0, result.Duration, result.ErrorMessage); + result.Success, null, result.BaselinedRecords, result.BaselinedRecords, 0, result.Duration, result.ErrorMessage); return result; } private async Task BaselineCoreAsync(BaselineRequest request) { var startTime = DateTime.UtcNow; + int baselinedCount = 0; + int baselinedRecords = 0; + // The file and target whose record is being written, so that a failure completes the file it started + (MigrationFileInfo MigrationFile, string TargetAlias, int RecordId, DateTime StartedAt)? baselineInProgress = null; try { @@ -1652,7 +1688,6 @@ private async Task BaselineCoreAsync(BaselineRequest request) filesToBaseline.Count, releaseLabel); // --- Phase 4: Record each file as migrated (without executing SQL) --- - int baselinedCount = 0; // Release-based ordering: Release → TargetGroup → Targets var baselineOrderedReleases = filesToBaseline @@ -1722,7 +1757,6 @@ async Task BaselineFile(MigrationFileInfo file, TargetOptions targetOptions) } var baselineStart = DateTime.UtcNow; - ReportFileStarted(MigrationOperation.Baseline, file, targetOptions.Alias!, 0, 0, baselineStart); // Validate CLI tool alias if set (no execution, but ensures config is correct for future rollbacks) string? cliAlias = ResolveUseCliToolAlias(file, targetOptions); @@ -1754,8 +1788,13 @@ async Task BaselineFile(MigrationFileInfo file, TargetOptions targetOptions) if (existingMigrationRecordId > 0) migrationRecordId = existingMigrationRecordId; + baselineInProgress = (file, targetOptions.Alias!, migrationRecordId, baselineStart); + ReportFileStarted(MigrationOperation.Baseline, file, targetOptions.Alias!, migrationRecordId, 0, baselineStart); + await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( migrationRecordId, MigrationStatus.Migrated, file.FileUpBlocksTotal)); + baselinedRecords++; + baselineInProgress = null; ReportFileCompleted(MigrationOperation.Baseline, file, targetOptions.Alias!, migrationRecordId, FileOutcome.Baselined, baselineStart); _logger.LogDebug("Baselined: {Filename} (Release: {Release}, Target: {Target})", @@ -1773,6 +1812,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, TargetReleaseVersion = request.TargetReleaseVersion, BaselinedFiles = baselinedCount, + BaselinedRecords = baselinedRecords, Duration = DateTime.UtcNow - startTime, Messages = new List { $"Successfully baselined {baselinedCount} migration file(s)" } }; @@ -1781,6 +1821,12 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( { _logger.LogError(ex, "Error during Baseline for product {Product}", request.ProductAlias); + if (baselineInProgress is { } inProgress) + { + ReportFileCompleted(MigrationOperation.Baseline, inProgress.MigrationFile, inProgress.TargetAlias, inProgress.RecordId, + FileOutcome.Failed, inProgress.StartedAt, ex.Message); + } + try { if (_ctxAccessor.Current.MigrationState.MigrationRunId > 0) @@ -1802,6 +1848,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( Environment = request.Environment, MigrationRunId = _ctxAccessor.Current.MigrationState.MigrationRunId, TargetReleaseVersion = request.TargetReleaseVersion, + BaselinedFiles = baselinedCount, + BaselinedRecords = baselinedRecords, Duration = DateTime.UtcNow - startTime, ErrorMessage = ex.Message, ErrorCode = ExtractErrorCode(ex), @@ -1844,6 +1892,28 @@ private async Task ExecuteRollbackForMigrations( foreach (var record in migrationsToRollback) { var fileStartTime = DateTime.UtcNow; + bool fileStartReported = false; + + // Raised once per record, before its completion. The execution path reports the parsed rollback file; + // the paths that end before the file is parsed report what the repository recorded for the record. + void ReportRollbackFileStarted(int blocksTotal, int resumeFromBlock) + { + if (fileStartReported) + { + return; + } + + fileStartReported = true; + ReportFileStarted(RollbackOperation(isErrorRecovery), record.ReleaseVersion, record.TargetGroupAlias, record.TargetAlias, + record.Filename, record.FileOrderId, record.Id, blocksTotal, resumeFromBlock, fileStartTime); + } + + void ReportRollbackFileCompleted(FileOutcome outcome, string? errorMessage) + { + ReportRollbackFileStarted(record.FileDownBlocksTotal ?? record.FileUpBlocksTotal, record.FileDownBlocksMigrated ?? 0); + ReportFileCompleted(RollbackOperation(isErrorRecovery), record.ReleaseVersion, record.TargetGroupAlias, record.TargetAlias, + record.Filename, record.Id, outcome, fileStartTime, errorMessage); + } try { @@ -1877,9 +1947,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( record.Id, MigrationStatus.Failed, record.FileUpBlocksMigrated)); } - ReportFileCompleted(RollbackOperation(isErrorRecovery), record.ReleaseVersion, record.TargetGroupAlias, record.TargetAlias, record.Filename, - - record.Id, FileOutcome.RollbackFailed, fileStartTime, $"Required rollback file not found: {rollbackFilename}"); + ReportRollbackFileCompleted(FileOutcome.RollbackFailed, $"Required rollback file not found: {rollbackFilename}"); result.AddFailure(record.Filename, $"Required rollback file not found: {rollbackFilename}", record.TargetAlias, record.Id); result.ErrorMessage = $"Rollback chain aborted: required rollback file missing for {record.Filename}"; @@ -1903,6 +1971,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( "Rollback stopped (StopRollbackOnMissingRollbackFile=true) | Product: {Product} | Env: {Environment} | Release: {Release} | TargetGroup: {TargetGroup} | Target: {Target} | File: {Filename} | MigrationRecordId: {MigrationRecordId} | Searched: {RollbackPath}", productOptions.Alias, _ctxAccessor.Current.RayMigratorConsoleOptions.Environment, record.ReleaseVersion, record.TargetGroupAlias, record.TargetAlias, record.Filename, record.Id, rollbackRelativePath); + ReportRollbackFileCompleted(FileOutcome.Skipped, $"Rollback stopped: rollback file not found: {rollbackFilename}"); result.AddWarning(record.Filename, $"Rollback stopped: rollback file not found: {rollbackFilename}"); return result; } @@ -1914,6 +1983,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( } // Don't update status — keep original (Migrated for successful migrations, Failed for failed ones) + ReportRollbackFileCompleted(FileOutcome.Skipped, $"Rollback file not found: {rollbackFilename}"); result.AddWarning(record.Filename, $"Rollback file not found: {rollbackFilename}"); result.SuccessCount++; continue; @@ -1960,8 +2030,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdateRollback( rollbackFileInfo.FileUpConfigJson)); } - ReportFileStarted(RollbackOperation(isErrorRecovery), record.ReleaseVersion, record.TargetGroupAlias, record.TargetAlias, - record.Filename, record.FileOrderId, record.Id, rollbackFileInfo.FileUpBlocksTotal, 0, fileStartTime); + ReportRollbackFileStarted(rollbackFileInfo.FileUpBlocksTotal, record.FileDownBlocksMigrated ?? 0); // Execute rollback SQL blocks bool fileHadBlockError = false; @@ -2054,9 +2123,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdateRollback( rollbackFileInfo.FileUpConfigJson)); } - ReportFileCompleted(RollbackOperation(isErrorRecovery), record.ReleaseVersion, record.TargetGroupAlias, record.TargetAlias, record.Filename, - - record.Id, FileOutcome.RollbackFailed, fileStartTime, $"Atomic rollback failed: {blockEx.Message}"); + ReportRollbackFileCompleted(FileOutcome.RollbackFailed, $"Atomic rollback failed: {blockEx.Message}"); result.AddFailure(record.Filename, $"Atomic rollback failed: {blockEx.Message}", record.TargetAlias, record.Id); result.ErrorMessage = $"Rollback chain aborted: atomic rollback of {record.Filename} failed"; @@ -2114,9 +2181,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdateRollback( rollbackFileInfo.FileUpBlocksTotal, rollbackFileInfo.FileUpConfigJson)); - ReportFileCompleted(RollbackOperation(isErrorRecovery), record.ReleaseVersion, record.TargetGroupAlias, record.TargetAlias, record.Filename, - - record.Id, FileOutcome.RollbackFailed, fileStartTime, $"Rollback block {blockIndex + 1} failed: {blockEx.Message}"); + ReportRollbackFileCompleted(FileOutcome.RollbackFailed, $"Rollback block {blockIndex + 1} failed: {blockEx.Message}"); result.AddFailure(record.Filename, $"Rollback block {blockIndex + 1} failed: {blockEx.Message}", record.TargetAlias, record.Id); result.ErrorMessage = $"Rollback chain aborted: block {blockIndex + 1} of {record.Filename} failed"; @@ -2181,8 +2246,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdateRollback( result.SuccessCount++; } - ReportFileCompleted(RollbackOperation(isErrorRecovery), record.ReleaseVersion, record.TargetGroupAlias, record.TargetAlias, record.Filename, - record.Id, fileHadBlockError ? FileOutcome.RollbackFailed : FileOutcome.RolledBack, fileStartTime, + ReportRollbackFileCompleted(fileHadBlockError ? FileOutcome.RollbackFailed : FileOutcome.RolledBack, fileHadBlockError ? "Rollback completed with errors (some blocks failed, RollbackErrorAction=Ignore)" : null); result.FileResults.Add(new MigrationFileResult @@ -2208,9 +2272,7 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdateRollback( "CRITICAL: Rollback failed for migration {MigrationRecordId} ({Filename}). Rollback chain ABORTED.", record.Id, record.Filename); - ReportFileCompleted(RollbackOperation(isErrorRecovery), record.ReleaseVersion, record.TargetGroupAlias, record.TargetAlias, record.Filename, - - record.Id, FileOutcome.RollbackFailed, fileStartTime, ex.Message); + ReportRollbackFileCompleted(FileOutcome.RollbackFailed, ex.Message); result.AddFailure(record.Filename, ex.Message, record.TargetAlias, record.Id); result.ErrorMessage = $"Rollback chain aborted at {record.Filename}: {ex.Message}"; @@ -3326,6 +3388,8 @@ await Task.Run(() => _templateExecutor.RepositoryMigrationUpdate( while (true) { int succeededBlocks = 0; + // Block events are raised after the commit: a transient error rolls the whole batch back and retries it. + var committedBlocks = new List(); await using var connection = targetDal.CreateConnection(); await connection.OpenAsync(); @@ -3355,7 +3419,7 @@ await targetDal.ExecuteNonQueryAsync( _templateExecutor.RepositoryMigrationUpdate( migrationRecordId, MigrationStatus.Executing, blockIndex + 1, connection, transaction, repoTimeout); - ReportBlockCompleted(MigrationOperation.MigrateUp, file, _ctxAccessor.Current.MigrationState.TargetAlias, migrationRecordId, blockIndex + 1, success: true); + committedBlocks.Add(blockIndex + 1); } // Final status update INSIDE the transaction (this is what closes the atomicity gap) @@ -3368,6 +3432,11 @@ await targetDal.ExecuteNonQueryAsync( await transaction.CommitAsync(); _ctxAccessor.Current.MigrationState.FileBlocksCommitted = file.FileUpBlocksTotal; + foreach (int blockNumber in committedBlocks) + { + ReportBlockCompleted(MigrationOperation.MigrateUp, file, _ctxAccessor.Current.MigrationState.TargetAlias, migrationRecordId, blockNumber, success: true); + } + return (succeededBlocks, 0, true); } catch (Exception ex) @@ -3414,6 +3483,9 @@ private async Task ExecuteRollbackBlocksAtomic( while (true) { + // Block events are raised after the commit: a transient error rolls the whole batch back and retries it. + var committedBlocks = new List(); + await using var connection = targetDal.CreateConnection(); await connection.OpenAsync(); await using var transaction = await connection.BeginTransactionAsync(); @@ -3448,7 +3520,7 @@ await targetDal.ExecuteNonQueryAsync( rollbackFileInfo.FileUpBlocksTotal, rollbackFileInfo.FileUpConfigJson, connection, transaction, repoTimeout); - ReportBlockCompleted(operation, record, rollbackFileInfo.FileUpBlocksTotal, blockIndex + 1, true, null); + committedBlocks.Add(blockIndex + 1); } // Final status update INSIDE the transaction (this is what closes the atomicity gap) @@ -3467,6 +3539,11 @@ await targetDal.ExecuteNonQueryAsync( } await transaction.CommitAsync(); + foreach (int blockNumber in committedBlocks) + { + ReportBlockCompleted(operation, record, rollbackFileInfo.FileUpBlocksTotal, blockNumber, true, null); + } + return; } catch (Exception ex) diff --git a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqliteProgressEventsTests.cs b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqliteProgressEventsTests.cs index fc2dd11..25ec1a3 100644 --- a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqliteProgressEventsTests.cs +++ b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqliteProgressEventsTests.cs @@ -83,6 +83,52 @@ public async Task Simulate_ReportsSimulatedOutcomesWithoutARunRow() recorder.Of().Should().ContainSingle().Which.Success.Should().BeTrue(); } + [Fact] + public async Task Simulate_ReportsTheReleaseTheRunStartsFrom() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + await ctx.MigrateUpAsync("Release_1.0"); + ctx.AssertSuccess(true); + + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + await ctx.MigrateUpAsync(runMode: MigrationRunMode.Simulate); + ctx.AssertSuccess(true); + + var started = recorder.Of().Should().ContainSingle().Which; + started.MigrationRunId.Should().Be(0, "Simulate writes no run row"); + started.FromRelease.Should().Be("Release_1.0", "a run without a run row still reports where the product stands"); + } + + [Fact] + public async Task MigrateDown_WithoutRollbackFile_ReportsTheRecordAsSkipped() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario() + .RemoveRollback("Release_4.0", "02_CreateTableH.sql") + .WithRequireRollbackFile(false) + .BuildAsync(); + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + await ctx.RebuildForAsync(MigrationCommand.MigrateDown, MigrationRunMode.Migrate, "Release_3.0"); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + await ctx.MigrateDownAsync("Release_3.0"); + ctx.AssertSuccess(true); + + var ends = recorder.Of(); + var skipped = ends.Where(e => e.Outcome == FileOutcome.Skipped).Should().ContainSingle("the record without a rollback file is skipped, not rolled back").Which; + skipped.FileName.Should().Be("02_CreateTableH.sql"); + skipped.MigrationRecordId.Should().BePositive(); + skipped.ErrorMessage.Should().Contain("not found"); + ends.Where(e => e.Outcome == FileOutcome.RolledBack).Select(e => e.FileName).Should().BeEquivalentTo(["01_CreateTableG.sql", "03_SeedDataD.sql"]); + recorder.Of().Select(s => (s.FileName, s.TargetAlias, s.MigrationRecordId)) + .Should().BeEquivalentTo(ends.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), "every completion has a start, the skipped record included"); + } + [Fact] public async Task MigrateDown_ReportsRollbacksOfTheRecordsAboveTheTargetRelease() { @@ -103,7 +149,13 @@ public async Task MigrateDown_ReportsRollbacksOfTheRecordsAboveTheTargetRelease( ends.Should().HaveCount(result.TotalMigrations); ends.Should().OnlyContain(e => e.Operation == MigrationOperation.MigrateDown && e.Outcome == FileOutcome.RolledBack && e.MigrationRecordId > 0); ends.Select(e => e.ReleaseVersion).Distinct().Should().BeEquivalentTo(["Release_3.0", "Release_4.0"]); - recorder.Of().Should().HaveCount(ends.Count); + var starts = recorder.Of(); + starts.Select(s => (s.FileName, s.TargetAlias, s.MigrationRecordId)) + .Should().BeEquivalentTo(ends.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), "every rollback completion has a start"); + starts.Should().OnlyContain(s => s.Operation == MigrationOperation.MigrateDown && s.BlocksTotal >= 1 && s.ResumeFromBlock == 0); + var blocks = recorder.Of(); + blocks.Should().NotBeEmpty("the rollback files have SQL blocks"); + blocks.Select(b => b.FileName).Distinct().Should().BeEquivalentTo(starts.Select(s => s.FileName).Distinct()); recorder.Of().Should().ContainSingle().Which.Success.Should().BeTrue(); } @@ -119,10 +171,13 @@ public async Task Baseline_ReportsBaselinedFilesWithoutBlocks() ctx.AssertSuccess(true); recorder.Of().Should().ContainSingle().Which.Command.Should().Be(MigrationCommand.Baseline); - recorder.Of().Should().HaveCount(result.BaselinedFiles) + var ends = recorder.Of(); + ends.Should().HaveCount(result.BaselinedRecords, "one completion per record written") .And.OnlyContain(e => e.Operation == MigrationOperation.Baseline && e.Outcome == FileOutcome.Baselined && e.MigrationRecordId > 0); + recorder.Of().Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)) + .Should().BeEquivalentTo(ends.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), "a start carries the record id as well"); recorder.Of().Should().BeEmpty("baseline executes no SQL"); - recorder.Of().Should().ContainSingle().Which.TotalFiles.Should().Be(result.BaselinedFiles); + recorder.Of().Should().ContainSingle().Which.TotalFiles.Should().Be(result.BaselinedRecords); } [Fact] From 87c28cdb6538f9ee023da62cd0ebb7245221071d Mon Sep 17 00:00:00 2001 From: danielpavic Date: Mon, 12 Oct 2026 23:56:05 +0200 Subject: [PATCH 09/14] Refuse a request whose environment differs from the context The repository ids, the run row and the settings snapshot take the environment from the MigrationContext, while file discovery and the environment-specific migsettings take it from the request, so a host that let them diverge ran one environment's files against another environment's rows. MigrateUpAsync, MigrateDownAsync and BaselineAsync now check the environment like the run mode and the target release. AllowOutOfOrder is decided by the request alone; the settings snapshot of a migrate-up run now records the request value instead of the console options, which a host may not have set. --- CHANGELOG.md | 11 +++++--- Docs/02-core-concepts/migration-context.md | 2 +- .../MigrationService.cs | 27 ++++++++++++++++--- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d8ad07..17d273c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,10 +59,13 @@ RayMigrator follows Semantic Versioning where applicable. carries `Environment` and `FromRelease`; `MigrationHistory`, `MigrationStatusInfo` and `BaselineResult` carry `Environment`. - `MigrateUpAsync`, `MigrateDownAsync` and `BaselineAsync` refuse a request - whose `TargetReleaseVersion` differs from the `MigrationContext`, like the - run-mode check does: the run row stamps the target release from the - context, so a host that let them diverge recorded a release the run never - aimed at. + whose `TargetReleaseVersion` or `Environment` differs from the + `MigrationContext`, like the run-mode check does: the run row stamps the + target release and the environment from the context, so a host that let + them diverge recorded a release the run never aimed at, or ran one + environment's files against another environment's rows. The settings + snapshot of a migrate-up run records the `AllowOutOfOrder` value of the + request, which is the value the run used. - `MigrationService` reads the configuration of the current run from its `MigrationContext` instead of from `IOptions`, and `TemplateCache` no longer takes the options: a host builds the cache once diff --git a/Docs/02-core-concepts/migration-context.md b/Docs/02-core-concepts/migration-context.md index c209d5c..9f34291 100644 --- a/Docs/02-core-concepts/migration-context.md +++ b/Docs/02-core-concepts/migration-context.md @@ -185,7 +185,7 @@ A host (RayMigrator Studio, an integration test) runs one operation per context. 2. Open the masking scope for the async flow of the run: `using var scope = SensitiveDataMasker.BeginScope(revealSensitiveData);`. Without an active scope or global initialization the masker is off and the MigrationRun settings snapshot stores connection strings unmasked. 3. `factory.Create(options, consoleOptions)`. The returned context is current for the calling async flow; set `context.ProgressObserver` before the operation if the host wants progress events. 4. Optionally `ConnectionValidator.ValidateTargetConnections(context, logger)`: it opens the connections the command's profile needs and fails fast with a clear message. -5. Call the operation on a scoped `IMigrationService`. The request must agree with the context: `MigrateUpAsync`, `MigrateDownAsync` and `BaselineAsync` refuse a request whose run mode or target release differs from the console options of the context (#6, #17). +5. Call the operation on a scoped `IMigrationService`. The request must agree with the context: `MigrateUpAsync`, `MigrateDownAsync` and `BaselineAsync` refuse a request whose run mode, target release or environment differs from the console options of the context (#6, #17). 6. Dispose the scope. The next run gets a fresh scope, a fresh context and a fresh `TemplateExecutor` (it caches the repository of the first context it sees, so it is registered as scoped). Per process, not per run: one `TemplateCache` (`new TemplateCache(revealSensitiveData, logger)`, then `ValidateConfigurationAgainstTemplateCache(options)` for each configuration the host runs), the `DatabaseLogWriter` and its Serilog sink when database logging is wanted, and the Serilog logger with `MigrationContextEnricher`, which reads `MigrationLoggingContext.Current` and stamps every event with run, release, target group, target, file and block. diff --git a/Raycoon.RayMigrator.Services/MigrationService.cs b/Raycoon.RayMigrator.Services/MigrationService.cs index 18c401d..53f3319 100644 --- a/Raycoon.RayMigrator.Services/MigrationService.cs +++ b/Raycoon.RayMigrator.Services/MigrationService.cs @@ -93,6 +93,7 @@ private async Task MigrateUpCoreAsync(MigrateUpRequest } EnsureRequestRunModeMatchesContext(request.RunMode); EnsureRequestTargetReleaseMatchesContext(request.TargetReleaseVersion); + EnsureRequestEnvironmentMatchesContext(request.Environment); // --- Phase 1: Initialization --- _ctxAccessor.Current.MigrationState.MigrationRunResult = MigrationRunResult.Running; @@ -117,7 +118,8 @@ private async Task MigrateUpCoreAsync(MigrateUpRequest } // Create migration run with settings snapshot (auto-fixes orphaned runs if needed) - var settingsJson = BuildMigrationRunSettingsJson(_ctxAccessor.Current); + // The request decides AllowOutOfOrder, so the snapshot records its value + var settingsJson = BuildMigrationRunSettingsJson(_ctxAccessor.Current, request.AllowOutOfOrder); _logger.LogTrace("MigrationRun settings snapshot:\n{SettingsJson}", settingsJson); await RepositoryMigrationRunInsertWithAutoFix(settingsJson); } @@ -1088,6 +1090,23 @@ private void EnsureRequestRunModeMatchesContext(MigrationRunMode requestRunMode) } } + /// + /// The repository ids, the run row and the settings snapshot take the environment from the + /// , while file discovery and the environment-specific migsettings take it from the + /// request. A host that let them diverge would run one environment's files against another environment's rows. + /// + /// The request environment differs from the context. + private void EnsureRequestEnvironmentMatchesContext(string requestEnvironment) + { + string? contextEnvironment = _ctxAccessor.Current.RayMigratorConsoleOptions.Environment; + if (!string.Equals(requestEnvironment, contextEnvironment, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Environment mismatch: the MigrationContext was built for [{contextEnvironment}] but the request names [{requestEnvironment}]. " + + "Build the context for the environment the request is executed with."); + } + } + /// /// The run row stamps ToReleaseVersion from the , while the execution path /// reads the target release from the request. The CLI fills both from the same option; a programmatic caller @@ -1313,6 +1332,7 @@ private async Task MigrateDownCoreAsync(MigrateDownReq } EnsureRequestRunModeMatchesContext(request.RunMode); EnsureRequestTargetReleaseMatchesContext(request.TargetReleaseVersion); + EnsureRequestEnvironmentMatchesContext(request.Environment); // --- Validate mode: validate rollback file existence and parseability --- if (!request.RunMode.ShouldReadRepository()) @@ -1603,6 +1623,7 @@ private async Task BaselineCoreAsync(BaselineRequest request) } EnsureRequestTargetReleaseMatchesContext(request.TargetReleaseVersion); + EnsureRequestEnvironmentMatchesContext(request.Environment); // --- Phase 1: Initialization --- await InitializeRepositoryAsync(); @@ -2926,7 +2947,7 @@ private static string SerializeTomlAsJson(bool useTransaction, string descriptio /// Builds a JSON snapshot of all RayMigrator settings at migration start. /// Connection strings are masked via SensitiveDataMasker.Mask() (respects RevealSensitiveData). /// - internal static string BuildMigrationRunSettingsJson(MigrationContext ctx) + internal static string BuildMigrationRunSettingsJson(MigrationContext ctx, bool? allowOutOfOrder = null) { var consoleOpts = ctx.RayMigratorConsoleOptions; var rayOpts = ctx.RayMigratorOptions; @@ -2945,7 +2966,7 @@ internal static string BuildMigrationRunSettingsJson(MigrationContext ctx) ["HashValidationScope"] = consoleOpts.HashValidationScope?.ToString(), ["ShowStartupInfo"] = consoleOpts.ShowStartupInfo, ["RevealSensitiveData"] = consoleOpts.RevealSensitiveData, - ["AllowOutOfOrder"] = consoleOpts.AllowOutOfOrder, + ["AllowOutOfOrder"] = allowOutOfOrder ?? consoleOpts.AllowOutOfOrder, ["FixScope"] = consoleOpts.FixScope?.ToString() }, ["Repository"] = new Dictionary From f2cca49d8b19b4e9d313f6f697c0682ddd26eebb Mon Sep 17 00:00:00 2001 From: danielpavic Date: Mon, 12 Oct 2026 23:56:05 +0200 Subject: [PATCH 10/14] Report the transient error codes from the context factory MigrationContextFactory replaced the pipeline's post-build steps but dropped the report of each DAL's transient error codes (ADR-022), so a TransientErrorCodes.txt that replaced the built-in list, and an empty list that disables every retry, went unlogged; the pipeline's helper had become dead code. The factory now logs the source when it adds the DAL properties, and makes the context current before its first log line, so the enricher stamps the schema-name warning with the run properties. The interface documentation names the actual schema-name contract: a missing name is an error only where the database requires one, a name on a database without schemas is a warning. --- .../transient-error-codes.md | 2 +- .../IMigrationContextFactory.cs | 7 +-- .../DirectModePipeline.cs | 32 ------------- .../MigrationContextFactory.cs | 46 +++++++++++++++++-- 4 files changed, 48 insertions(+), 39 deletions(-) diff --git a/Docs/03-database-layer/transient-error-codes.md b/Docs/03-database-layer/transient-error-codes.md index 6f8dcb5..1124607 100644 --- a/Docs/03-database-layer/transient-error-codes.md +++ b/Docs/03-database-layer/transient-error-codes.md @@ -36,7 +36,7 @@ Every DAL retries a database command when the provider reports an error it class - **Missing file.** The DAL keeps its built-in list (`DalBase.DefaultTransientErrorCodes`). Delete the file to return to the defaults. - **Empty file.** An existing file without codes means that no database error is retried; RayMigrator logs a Warning at start. - **When it is read.** `DalFactory` reads the file once when it creates the DAL instance, i.e. at application start. A change needs a restart. -- **Logging.** `DirectModePipeline` logs the source once per DatabaseType: Information when the file was loaded (the full path only with `--reveal-sensitive-data`), Debug for the built-in list, Warning when the effective list is empty. +- **Logging.** `MigrationContextFactory` logs the source once per DatabaseType when it fills the DAL properties of a context: Information when the file was loaded (the full path only with `--reveal-sensitive-data`), Debug for the built-in list, Warning when the effective list is empty. - **Upgrades.** Every publish or upgrade overwrites `DataAccessLayers/`, so edits must be re-applied after an update, exactly like edited SQL templates. ## Built-in lists diff --git a/Raycoon.RayMigrator.Core/IMigrationContextFactory.cs b/Raycoon.RayMigrator.Core/IMigrationContextFactory.cs index e92399d..431c3fe 100644 --- a/Raycoon.RayMigrator.Core/IMigrationContextFactory.cs +++ b/Raycoon.RayMigrator.Core/IMigrationContextFactory.cs @@ -27,7 +27,8 @@ public interface IMigrationContextFactory /// /// /// The options do not pass preparation, the product alias is unknown, a configured DatabaseType has no - /// data access layer, or a schema name is set for a database that does not support schemas. + /// data access layer, or a schema name is missing where the database requires one. A schema name on a database + /// without schemas is only logged as a warning. /// MigrationContext Create(RayMigratorOptions options, RayMigratorConsoleOptions consoleOptions); @@ -36,8 +37,8 @@ public interface IMigrationContextFactory /// (the CLI pipeline). Everything else is as in . /// /// - /// The product alias is unknown, a configured DatabaseType has no data access layer, or a schema name is set - /// for a database that does not support schemas. + /// The product alias is unknown, a configured DatabaseType has no data access layer, or a schema name is + /// missing where the database requires one. /// MigrationContext CreateFromValidatedOptions(RayMigratorOptions options, RayMigratorConsoleOptions consoleOptions); } diff --git a/Raycoon.RayMigrator.Pipeline/DirectModePipeline.cs b/Raycoon.RayMigrator.Pipeline/DirectModePipeline.cs index 0151369..7d1dbca 100644 --- a/Raycoon.RayMigrator.Pipeline/DirectModePipeline.cs +++ b/Raycoon.RayMigrator.Pipeline/DirectModePipeline.cs @@ -490,36 +490,4 @@ private static DatabaseLogWriter CreateDatabaseLogWriter( } } - /// - /// Reports where a DAL's transient error codes come from (ADR-022): Information when a TransientErrorCodes.txt - /// replaced the built-in list (the path only with RevealSensitiveData), Debug for the built-in list, and a Warning - /// when the effective list is empty, because then no database error is retried. - /// - private static void LogTransientErrorCodes(DalBase dal, string databaseType, bool revealSensitiveData, Microsoft.Extensions.Logging.ILogger logger) - { - string codes = string.Join(", ", dal.TransientErrorCodes); - // The file path reveals the installation folder; show it only when sensitive data may be revealed. - string source = revealSensitiveData && dal.TransientErrorCodesSource != null - ? dal.TransientErrorCodesSource - : TransientErrorCodesFile.FileName; - if (dal.TransientErrorCodesSource != null) - { - logger.LogInformation(MigrationEvent.InitializeDalSpecificProperties, - "Transient error codes for DatabaseType [{DatabaseType}] loaded from [{Source}]: [{Codes}]", - databaseType, source, codes); - } - else - { - logger.LogDebug(MigrationEvent.InitializeDalSpecificProperties, - "Transient error codes for DatabaseType [{DatabaseType}] are the built-in list: [{Codes}]", - databaseType, codes); - } - - if (dal.TransientErrorCodes.Count == 0) - { - logger.LogWarning(MigrationEvent.InitializeDalSpecificProperties, - "No transient error codes for DatabaseType [{DatabaseType}]: no database error is retried. Add codes to [{Source}] or delete the file to restore the built-in list", - databaseType, source); - } - } } diff --git a/Raycoon.RayMigrator.Services/MigrationContextFactory.cs b/Raycoon.RayMigrator.Services/MigrationContextFactory.cs index 835bee5..27107fd 100644 --- a/Raycoon.RayMigrator.Services/MigrationContextFactory.cs +++ b/Raycoon.RayMigrator.Services/MigrationContextFactory.cs @@ -19,6 +19,7 @@ public sealed class MigrationContextFactory : IMigrationContextFactory private readonly IMigrationContextAccessor _ctxAccessor; private readonly ILogger _logger; + /// Creates the factory. Every context it creates becomes current through . public MigrationContextFactory(IMigrationContextAccessor ctxAccessor, ILogger logger) { _ctxAccessor = ctxAccessor; @@ -51,6 +52,10 @@ private MigrationContext Activate(MigrationContext ctx) { var options = ctx.RayMigratorOptions; + // Current first: the enricher then stamps the log lines below with the run properties of this context + _ctxAccessor.Current = ctx; + MigrationLoggingContext.Current = ctx; + _logger.LogDebug(MigrationEvent.InitializeDalSpecificProperties, "Initializing DalSpecificPropertiesDictionary..."); SetDalSpecificProperties(ctx, options.Repository!.DatabaseType!, options.Repository.ConnectionString!); @@ -83,9 +88,6 @@ private MigrationContext Activate(MigrationContext ctx) "DatabaseLogging", _logger); } - _ctxAccessor.Current = ctx; - MigrationLoggingContext.Current = ctx; - return ctx; } @@ -107,6 +109,44 @@ private void SetDalSpecificProperties(MigrationContext ctx, string databaseType, string.IsNullOrWhiteSpace(dalInstance.DalSpecificProperties.SqlBlockDelimiter) ? "{Empty}" : dalInstance.DalSpecificProperties.SqlBlockDelimiter, dalInstance.DalSpecificProperties.SupportsSchema, dalInstance.DalSpecificProperties.SupportsTransactionalDdl); + + if (dalInstance is DalBase dalBase) + { + LogTransientErrorCodes(dalBase, databaseType, ctx.RayMigratorConsoleOptions.RevealSensitiveData); + } + } + } + + /// + /// Reports where a DAL's transient error codes come from (ADR-022): Information when a TransientErrorCodes.txt + /// replaced the built-in list (the path only with RevealSensitiveData), Debug for the built-in list, and a Warning + /// when the effective list is empty, because then no database error is retried. + /// + private void LogTransientErrorCodes(DalBase dal, string databaseType, bool revealSensitiveData) + { + string codes = string.Join(", ", dal.TransientErrorCodes); + // The file path reveals the installation folder; show it only when sensitive data may be revealed. + string source = revealSensitiveData && dal.TransientErrorCodesSource != null + ? dal.TransientErrorCodesSource + : TransientErrorCodesFile.FileName; + if (dal.TransientErrorCodesSource != null) + { + _logger.LogInformation(MigrationEvent.InitializeDalSpecificProperties, + "Transient error codes for DatabaseType [{DatabaseType}] loaded from [{Source}]: [{Codes}]", + databaseType, source, codes); + } + else + { + _logger.LogDebug(MigrationEvent.InitializeDalSpecificProperties, + "Transient error codes for DatabaseType [{DatabaseType}] are the built-in list: [{Codes}]", + databaseType, codes); + } + + if (dal.TransientErrorCodes.Count == 0) + { + _logger.LogWarning(MigrationEvent.InitializeDalSpecificProperties, + "No transient error codes for DatabaseType [{DatabaseType}]: no database error is retried. Add codes to [{Source}] or delete the file to restore the built-in list", + databaseType, source); } } } From 8abba73f5319cd33adb1486233e20ec5242b4099 Mon Sep 17 00:00:00 2001 From: danielpavic Date: Mon, 12 Oct 2026 23:56:05 +0200 Subject: [PATCH 11/14] Check placeholder connection strings as written in the preparer RayMigratorOptionsPreparer resolved the {ENV:...} placeholders in place and then ran the rule catalog without a section as written, so a host whose connection string supplied its password through a placeholder got the RULE_7_3 hint that the CLI stopped giving in #30. The preparer now builds the as-written section from the values it replaced, keyed by their configuration path, and the validator checks those as written and everything else as bound. --- CHANGELOG.md | 4 ++- .../Validation/RayMigratorOptionsPreparer.cs | 26 +++++++++++++++-- .../Raycoon.RayMigrator.Core.csproj | 1 + .../P1_RayMigratorOptionsPreparerTests.cs | 29 +++++++++++++++++++ 4 files changed, 57 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17d273c..8196774 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,9 @@ RayMigrator follows Semantic Versioning where applicable. engine knows its own). The accessor is a singleton in both host modes; the API accessor isolates per async flow through its `AsyncLocal` anyway. `ProductAliasGuard` explains an unknown product alias with the same message - the CLI showed, now also for hosts. + the CLI showed, now also for hosts. The preparer checks connection strings + that contain placeholders as they were written, so a credential a + placeholder supplies is not reported as hardcoded (#30). - The result contracts of `IMigrationService` name the repository run and the target. `MigrationOperationResult`, `BaselineResult` and `MigrationRunInfo` carry `MigrationRunId`, the id of the run row (0 when the run mode writes no diff --git a/Raycoon.RayMigrator.Core/Configuration/Validation/RayMigratorOptionsPreparer.cs b/Raycoon.RayMigrator.Core/Configuration/Validation/RayMigratorOptionsPreparer.cs index ac95036..6cc101f 100644 --- a/Raycoon.RayMigrator.Core/Configuration/Validation/RayMigratorOptionsPreparer.cs +++ b/Raycoon.RayMigrator.Core/Configuration/Validation/RayMigratorOptionsPreparer.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Options; using Raycoon.RayMigrator.Core.Configuration.Options; using Raycoon.RayMigrator.Core.Configuration.Replacer; +using Raycoon.RayMigrator.Shared.Constants; using Raycoon.RayMigrator.Shared.Exceptions; namespace Raycoon.RayMigrator.Core.Configuration.Validation; @@ -22,7 +23,8 @@ public static class RayMigratorOptionsPreparer /// /// The configuration as written, before placeholders were resolved, when the host has one. RULE_7_3 then /// inspects the connection strings as written, so a credential that enters through a placeholder is not - /// reported as hardcoded (#30). Null checks the values as they are. + /// reported as hardcoded (#30). Without one, the values that contained placeholders are checked as they were + /// written and everything else as bound. /// /// /// A placeholder names an environment variable that is not set or empty, an enum-typed value is not an @@ -53,13 +55,33 @@ public static OptionsPreparationResult Prepare(RayMigratorOptions options, IConf new ProductDefaultsPostConfigureOptions().PostConfigure(null, options); ThrowIfFailed(new DataAnnotationValidateOptions(null).Validate(null, options), "Configuration validation failed"); - ThrowIfFailed(new RayMigratorOptionsValidator(sectionAsWritten).Validate(null, options), "Configuration validation failed"); + ThrowIfFailed(new RayMigratorOptionsValidator(sectionAsWritten ?? AsWrittenSection(replaced)).Validate(null, options), "Configuration validation failed"); SensitiveDataMasker.RegisterSensitiveData(options); return new OptionsPreparationResult(replaced); } + /// + /// The values that contained placeholders, as they were written, so that RULE_7_3 sees {ENV:...} and not + /// the resolved credential (#30). Every other key falls back to the bound value. Null when nothing was replaced. + /// + internal static IConfigurationSection? AsWrittenSection(IEnumerable replaced) + { + var values = replaced + .GroupBy(r => r.Path) + .ToDictionary(g => g.Key, g => g.First().ConfigurationValue); + if (values.Count == 0) + { + return null; + } + + return new ConfigurationBuilder() + .AddInMemoryCollection(values) + .Build() + .GetSection(InternalConstants.RayMigratorSectionName); + } + private static void ThrowIfFailed(ValidateOptionsResult result, string prefix) { if (!result.Failed) diff --git a/Raycoon.RayMigrator.Core/Raycoon.RayMigrator.Core.csproj b/Raycoon.RayMigrator.Core/Raycoon.RayMigrator.Core.csproj index 0581988..485506c 100644 --- a/Raycoon.RayMigrator.Core/Raycoon.RayMigrator.Core.csproj +++ b/Raycoon.RayMigrator.Core/Raycoon.RayMigrator.Core.csproj @@ -22,6 +22,7 @@ + diff --git a/Raycoon.RayMigrator.Tests.Unit/P1_RayMigratorOptionsPreparerTests.cs b/Raycoon.RayMigrator.Tests.Unit/P1_RayMigratorOptionsPreparerTests.cs index b25e419..4e03b37 100644 --- a/Raycoon.RayMigrator.Tests.Unit/P1_RayMigratorOptionsPreparerTests.cs +++ b/Raycoon.RayMigrator.Tests.Unit/P1_RayMigratorOptionsPreparerTests.cs @@ -6,6 +6,7 @@ using Raycoon.RayMigrator.Core.Configuration.Validation; using Raycoon.RayMigrator.Shared.Exceptions; using Raycoon.RayMigrator.Tests.Unit.Helpers; +using Raycoon.RayMigrator.Validation; namespace Raycoon.RayMigrator.Tests.Unit; @@ -110,9 +111,33 @@ public void Prepare_ResolvesPlaceholdersAndRegistersTheValuesWithTheMasker() .And.NotContain("Data Source=:memory:", "the connection strings of the options are registered as well"); } + [Fact] + public void Prepare_CredentialFromPlaceholder_IsNotReportedAsHardcoded() + { + using var maskerScope = SensitiveDataMasker.BeginScope(revealSensitiveData: false); + string pwdVar = SetVariable("s3cr3t"); + var options = CreateOptions($"Data Source={_dir}/target.db;Password={{ENV:{pwdVar}}}"); + string asWritten = options.Products![0].TargetGroups![0].Targets![0].ConnectionString!; + + var replaced = EnvironmentVariableReplacer.ReplaceWithEnvironmentVariables(options); + var section = RayMigratorOptionsPreparer.AsWrittenSection(replaced); + + section.Should().NotBeNull(); + section!["Products:0:TargetGroups:0:Targets:0:ConnectionString"].Should().Be(asWritten, + "the rule catalog sees the value as written, with the placeholder"); + RayMigratorOptionsValidator.GetReport(options, section).Warnings.Should().NotContain(w => w.Code == RuleIds.RULE_7_3, + "the credential enters through a placeholder (#30)"); + RayMigratorOptionsValidator.GetReport(options).Warnings.Should().Contain(w => w.Code == RuleIds.RULE_7_3, + "the resolved value alone looks hardcoded, which is what a host without an as-written section got before"); + + var act = () => RayMigratorOptionsPreparer.Prepare(options); + act.Should().NotThrow(); + } + [Fact] public void Prepare_MissingVariable_ThrowsNamingVariableAndPath() { + using var maskerScope = SensitiveDataMasker.BeginScope(revealSensitiveData: false); var options = CreateOptions("Data Source={ENV:RM_PREP_DOES_NOT_EXIST}/target.db"); var act = () => RayMigratorOptionsPreparer.Prepare(options); @@ -125,6 +150,7 @@ public void Prepare_MissingVariable_ThrowsNamingVariableAndPath() [Fact] public void Prepare_WritesTheDocumentedDefaultsIntoNullProperties() { + using var maskerScope = SensitiveDataMasker.BeginScope(revealSensitiveData: false); var options = CreateOptions("Data Source=:memory:"); options.Repository!.DbCommandTimeoutInSeconds = null; options.Repository.DbCommandMaxRetries = null; @@ -147,6 +173,7 @@ public void Prepare_WritesTheDocumentedDefaultsIntoNullProperties() [Fact] public void Prepare_InvalidEnumValue_Throws() { + using var maskerScope = SensitiveDataMasker.BeginScope(revealSensitiveData: false); var options = CreateOptions("Data Source=:memory:"); options.Products![0].MigrationErrorAction = "Explode"; @@ -158,6 +185,7 @@ public void Prepare_InvalidEnumValue_Throws() [Fact] public void Prepare_RuleCatalogError_Throws() { + using var maskerScope = SensitiveDataMasker.BeginScope(revealSensitiveData: false); var options = CreateOptions("Data Source=:memory:"); var targets = options.Products![0].TargetGroups![0].Targets!; targets.Add(new TargetOptions { Alias = targets[0].Alias, ConnectionString = "Data Source=:memory:" }); @@ -170,6 +198,7 @@ public void Prepare_RuleCatalogError_Throws() [Fact] public void Prepare_Twice_LeavesTheOptionsUnchanged() { + using var maskerScope = SensitiveDataMasker.BeginScope(revealSensitiveData: false); string dirVar = SetVariable(_dir); var options = CreateOptions($"Data Source={{ENV:{dirVar}}}/target.db"); options.Repository!.DbCommandTimeoutInSeconds = null; From 071a5ac3dc13a37f4c61ec1a4169859255f294c0 Mon Sep 17 00:00:00 2001 From: danielpavic Date: Mon, 12 Oct 2026 23:56:05 +0200 Subject: [PATCH 12/14] Isolate the masker in the factory and guard unit tests The tests that create a context through the factory registered the options' secrets with the global SensitiveDataMasker, which runs in parallel with the masker tests and made them order-dependent. Each test now opens its own masker scope, as the preparer tests already did. --- .../P1_MigrationContextFactoryTests.cs | 7 +++++++ .../P1_MigrationContextGuardTests.cs | 3 +++ 2 files changed, 10 insertions(+) diff --git a/Raycoon.RayMigrator.Tests.Unit/P1_MigrationContextFactoryTests.cs b/Raycoon.RayMigrator.Tests.Unit/P1_MigrationContextFactoryTests.cs index 45f9b0a..fdc7c42 100644 --- a/Raycoon.RayMigrator.Tests.Unit/P1_MigrationContextFactoryTests.cs +++ b/Raycoon.RayMigrator.Tests.Unit/P1_MigrationContextFactoryTests.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Raycoon.RayMigrator.Core; +using Raycoon.RayMigrator.Core.Configuration; using Raycoon.RayMigrator.Core.Configuration.Enums; using Raycoon.RayMigrator.Core.Configuration.Options; using Raycoon.RayMigrator.Services; @@ -53,6 +54,7 @@ private static RayMigratorConsoleOptions ConsoleOptions(string product = Profile [Fact] public void CreateFromValidatedOptions_FillsDalPropertiesForRepositoryTargetsAndLogging() { + using var maskerScope = SensitiveDataMasker.BeginScope(revealSensitiveData: false); var options = CreateOptions(databaseLoggingConnectionString: "Data Source=:memory:"); options.Products![0].TargetGroups!.Add(new TargetGroupOptions { @@ -72,6 +74,7 @@ public void CreateFromValidatedOptions_FillsDalPropertiesForRepositoryTargetsAnd [Fact] public void Create_MakesTheContextCurrentForTheCallingFlow() { + using var maskerScope = SensitiveDataMasker.BeginScope(revealSensitiveData: false); var accessor = new AsyncLocalMigrationContextAccessor(); var factory = CreateFactory(accessor); @@ -87,6 +90,7 @@ public void Create_MakesTheContextCurrentForTheCallingFlow() [Fact] public void Create_PreparesTheOptions() { + using var maskerScope = SensitiveDataMasker.BeginScope(revealSensitiveData: false); string dirVar = "RM_CTXF_" + Guid.NewGuid().ToString("N")[..8].ToUpperInvariant(); Environment.SetEnvironmentVariable(dirVar, _dir); _variables.Add(dirVar); @@ -102,6 +106,7 @@ public void Create_PreparesTheOptions() [Fact] public void Create_UnknownProduct_ThrowsNamingTheAvailableAliases() { + using var maskerScope = SensitiveDataMasker.BeginScope(revealSensitiveData: false); var act = () => CreateFactory().Create(CreateOptions(), ConsoleOptions(product: "Nope")); act.Should().Throw() @@ -112,6 +117,7 @@ public void Create_UnknownProduct_ThrowsNamingTheAvailableAliases() [Fact] public void Create_ProductAliasWithWrongCase_HintsAtTheConfiguredAlias() { + using var maskerScope = SensitiveDataMasker.BeginScope(revealSensitiveData: false); var act = () => CreateFactory().Create(CreateOptions(), ConsoleOptions(product: ProfileTestContext.ProductAlias.ToUpperInvariant())); act.Should().Throw().WithMessage("*Did you mean*"); @@ -120,6 +126,7 @@ public void Create_ProductAliasWithWrongCase_HintsAtTheConfiguredAlias() [Fact] public void AddRayMigratorServices_ApiMode_ResolvesTheFactoryFromAScope() { + using var maskerScope = SensitiveDataMasker.BeginScope(revealSensitiveData: false); var services = new ServiceCollection(); services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); services.AddRayMigratorServices(RayMigratorHostMode.Api); diff --git a/Raycoon.RayMigrator.Tests.Unit/P1_MigrationContextGuardTests.cs b/Raycoon.RayMigrator.Tests.Unit/P1_MigrationContextGuardTests.cs index bbbac2e..3b03c95 100644 --- a/Raycoon.RayMigrator.Tests.Unit/P1_MigrationContextGuardTests.cs +++ b/Raycoon.RayMigrator.Tests.Unit/P1_MigrationContextGuardTests.cs @@ -1,6 +1,7 @@ using AwesomeAssertions; using Microsoft.Extensions.Logging.Abstractions; using Raycoon.RayMigrator.Core; +using Raycoon.RayMigrator.Core.Configuration; using Raycoon.RayMigrator.Core.Configuration.Enums; using Raycoon.RayMigrator.Core.Configuration.Options; using Raycoon.RayMigrator.Services; @@ -61,6 +62,7 @@ public void Constructor_WithValidCommandAndRunMode_Succeeds(MigrationCommand com [InlineData(MigrationCommand.MigrateUp, MigrationRunMode.Simulate)] public void Factory_StampsTheRequestedCommand(MigrationCommand command, MigrationRunMode runMode) { + using var maskerScope = SensitiveDataMasker.BeginScope(revealSensitiveData: false); var options = ProfileTestContext.CreateOptions("Data Source=:memory:"); var consoleOptions = new RayMigratorConsoleOptions @@ -80,6 +82,7 @@ public void Factory_StampsTheRequestedCommand(MigrationCommand command, Migratio [Fact] public void Factory_WithCommandNone_Throws() { + using var maskerScope = SensitiveDataMasker.BeginScope(revealSensitiveData: false); var options = ProfileTestContext.CreateOptions("Data Source=:memory:"); var consoleOptions = new RayMigratorConsoleOptions From f07bd863f1edf999acbf57074093f64a70103448 Mon Sep 17 00:00:00 2001 From: danielpavic Date: Mon, 12 Oct 2026 23:56:05 +0200 Subject: [PATCH 13/14] Cover history, FromReleaseVersion and progress events on every engine The per-environment history, the release a run starts from and the progress events were tested on SQLite only. The tests now run against PostgreSQL, SQL Server, MariaDB and MySQL as well, as the testing convention requires for engine behaviour. --- .../Tests/Features/InfoTests.cs | 34 +++ .../Tests/Features/MariaDbInfoTests.cs | 34 +++ .../Features/MariaDbMigrationRunMetaTests.cs | 19 ++ .../Features/MariaDbProgressEventsTests.cs | 231 ++++++++++++++++++ .../Tests/Features/MigrationRunMetaTests.cs | 19 ++ .../Tests/Features/MySqlInfoTests.cs | 34 +++ .../Features/MySqlMigrationRunMetaTests.cs | 19 ++ .../Features/MySqlProgressEventsTests.cs | 231 ++++++++++++++++++ .../Tests/Features/ProgressEventsTests.cs | 231 ++++++++++++++++++ .../Tests/Features/SqlServerInfoTests.cs | 34 +++ .../SqlServerMigrationRunMetaTests.cs | 19 ++ .../Features/SqlServerProgressEventsTests.cs | 231 ++++++++++++++++++ 12 files changed, 1136 insertions(+) create mode 100644 Raycoon.RayMigrator.Tests.Engine/Tests/Features/MariaDbProgressEventsTests.cs create mode 100644 Raycoon.RayMigrator.Tests.Engine/Tests/Features/MySqlProgressEventsTests.cs create mode 100644 Raycoon.RayMigrator.Tests.Engine/Tests/Features/ProgressEventsTests.cs create mode 100644 Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqlServerProgressEventsTests.cs diff --git a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/InfoTests.cs b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/InfoTests.cs index 8a2f79a..52fdfc2 100644 --- a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/InfoTests.cs +++ b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/InfoTests.cs @@ -326,4 +326,38 @@ public async Task GetHistory_AfterMigrateUpWithRollback_ShowsUpRunWithRolledBack history.Runs[0].SuccessfulMigrations.Should().Be(0, "every record was rolled back again"); history.Runs[0].FailedMigrations.Should().Be(0, "the failed file was rolled back too"); } + + /// + /// History and status are per environment: a run of another environment of the same product is not listed. + /// + [Fact] + public async Task GetHistory_ListsOnlyTheRunsOfTheRequestedEnvironment() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + + await using var ctx = await CreateScenario().BuildAsync(); + + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + // A second environment on the same repository: baseline writes a run row without touching the targets. + await ctx.RebuildForAsync(MigrationCommand.Baseline, MigrationRunMode.Migrate, environment: "Staging"); + await ctx.BaselineAsync(); + ctx.AssertSuccess(true); + ctx.AssertRunCount(2); + + await ctx.RebuildForAsync(MigrationCommand.Info, MigrationRunMode.Migrate, environment: "Staging"); + var staging = await ctx.GetHistoryAsync(); + staging.Environment.Should().Be("Staging"); + staging.Runs.Should().ContainSingle("the up run belongs to the Docker environment") + .Which.Operation.Should().Be(MigrationOperation.Baseline); + + await ctx.RebuildForAsync(MigrationCommand.Info, MigrationRunMode.Migrate); + var docker = await ctx.GetHistoryAsync(); + docker.Environment.Should().Be("Docker"); + docker.Runs.Should().ContainSingle("the baseline run belongs to the Staging environment") + .Which.Operation.Should().Be(MigrationOperation.MigrateUp); + docker.Runs[0].FromRelease.Should().BeNull("nothing was migrated before the first run"); + docker.Runs[0].MigrationRunId.Should().BePositive(); + } } diff --git a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MariaDbInfoTests.cs b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MariaDbInfoTests.cs index 12647ae..30984f7 100644 --- a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MariaDbInfoTests.cs +++ b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MariaDbInfoTests.cs @@ -326,4 +326,38 @@ public async Task GetHistory_AfterMigrateUpWithRollback_ShowsUpRunWithRolledBack history.Runs[0].SuccessfulMigrations.Should().Be(0, "every record was rolled back again"); history.Runs[0].FailedMigrations.Should().Be(0, "the failed file was rolled back too"); } + + /// + /// History and status are per environment: a run of another environment of the same product is not listed. + /// + [Fact] + public async Task GetHistory_ListsOnlyTheRunsOfTheRequestedEnvironment() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + + await using var ctx = await CreateScenario().BuildAsync(); + + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + // A second environment on the same repository: baseline writes a run row without touching the targets. + await ctx.RebuildForAsync(MigrationCommand.Baseline, MigrationRunMode.Migrate, environment: "Staging"); + await ctx.BaselineAsync(); + ctx.AssertSuccess(true); + ctx.AssertRunCount(2); + + await ctx.RebuildForAsync(MigrationCommand.Info, MigrationRunMode.Migrate, environment: "Staging"); + var staging = await ctx.GetHistoryAsync(); + staging.Environment.Should().Be("Staging"); + staging.Runs.Should().ContainSingle("the up run belongs to the Docker environment") + .Which.Operation.Should().Be(MigrationOperation.Baseline); + + await ctx.RebuildForAsync(MigrationCommand.Info, MigrationRunMode.Migrate); + var docker = await ctx.GetHistoryAsync(); + docker.Environment.Should().Be("Docker"); + docker.Runs.Should().ContainSingle("the baseline run belongs to the Staging environment") + .Which.Operation.Should().Be(MigrationOperation.MigrateUp); + docker.Runs[0].FromRelease.Should().BeNull("nothing was migrated before the first run"); + docker.Runs[0].MigrationRunId.Should().BePositive(); + } } diff --git a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MariaDbMigrationRunMetaTests.cs b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MariaDbMigrationRunMetaTests.cs index 4eea67c..58a67d9 100644 --- a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MariaDbMigrationRunMetaTests.cs +++ b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MariaDbMigrationRunMetaTests.cs @@ -232,4 +232,23 @@ public async Task Baseline_SettingsJson_ShouldContainMaskedConnectionStrings() } } + [Fact] + public async Task FromReleaseVersion_IsTheHighestMigratedReleaseWhenTheRunStarted() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + + // The run row stamps ToReleaseVersion from the context, so the host is rebuilt for each target release. + await ctx.RebuildForAsync(MigrationCommand.MigrateUp, MigrationRunMode.Migrate, "Release_2.0"); + await ctx.MigrateUpAsync("Release_2.0"); + ctx.AssertSuccess(true); + await ctx.RebuildForAsync(MigrationCommand.MigrateUp, MigrationRunMode.Migrate); + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + ctx.AssertRunCount(2); + ctx.GetMigrationRun(1)!.FromReleaseVersion.Should().BeNull("nothing was migrated before the first run"); + ctx.AssertMigrationRun(1, new MigrationRunExpectation { ToReleaseVersion = "Release_2.0" }); + ctx.AssertMigrationRun(2, new MigrationRunExpectation { FromReleaseVersion = "Release_2.0" }); + } } diff --git a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MariaDbProgressEventsTests.cs b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MariaDbProgressEventsTests.cs new file mode 100644 index 0000000..8f798ec --- /dev/null +++ b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MariaDbProgressEventsTests.cs @@ -0,0 +1,231 @@ +using Raycoon.RayMigrator.Core.Configuration.Enums; +using Raycoon.RayMigrator.Core.Progress; +using Raycoon.RayMigrator.Tests.Engine.Fixtures; +using Raycoon.RayMigrator.Tests.Engine.Infrastructure; + +namespace Raycoon.RayMigrator.Tests.Engine.Tests.Features; + +/// +/// The typed progress events a host observes through : one run +/// event pair per operation, one file event pair per file and target, one block event per executed block. +/// +[Collection("MariaDb")] +[Trait("Engine", "MariaDb")] +[Trait("Category", "Features")] +public class MariaDbProgressEventsTests : MariaDbTestBase +{ + public MariaDbProgressEventsTests(MariaDbFixture fixture) : base(fixture) { } + + [Fact] + public async Task MigrateUp_ReportsTheRunAndEveryFileOnEveryTarget() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.MigrateUpAsync("Release_1.0"); + ctx.AssertSuccess(true); + + recorder.Events.First().Should().BeOfType(); + recorder.Events.Last().Should().BeOfType(); + + var started = recorder.Of().Should().ContainSingle().Which; + started.Command.Should().Be(MigrationCommand.MigrateUp); + started.RunMode.Should().Be(MigrationRunMode.Migrate); + started.MigrationRunId.Should().Be(result.MigrationRunId, "the run row exists when the run is reported as started"); + started.MigrationRunId.Should().BePositive(); + started.FromRelease.Should().BeNull("nothing was migrated before the first run"); + started.TargetRelease.Should().Be("Release_1.0"); + + var completed = recorder.Of().Should().ContainSingle().Which; + completed.Success.Should().BeTrue(); + completed.Result.Should().Be(MigrationRunResult.Ok); + completed.MigrationRunId.Should().Be(result.MigrationRunId); + completed.TotalFiles.Should().Be(result.TotalMigrations); + + var fileStarts = recorder.Of(); + var fileEnds = recorder.Of(); + fileStarts.Should().HaveCount(result.TotalMigrations, "one start per file and target"); + fileEnds.Should().HaveCount(result.TotalMigrations, "one completion per file and target"); + fileEnds.Should().OnlyContain(e => e.Outcome == FileOutcome.Migrated && e.Operation == MigrationOperation.MigrateUp); + fileStarts.Should().OnlyContain(e => e.MigrationRecordId > 0 && e.ReleaseVersion == "Release_1.0" && e.BlocksTotal >= 1); + fileEnds.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)) + .Should().BeEquivalentTo(fileStarts.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), + "a completion carries the same identity as its start"); + + foreach (var end in fileEnds) + { + int startIndex = recorder.Events.IndexOf(fileStarts.Single(s => s.FileName == end.FileName && s.TargetAlias == end.TargetAlias)); + recorder.Events.IndexOf(end).Should().BeGreaterThan(startIndex, "a file completes after it started"); + } + + var blocks = recorder.Of(); + blocks.Should().NotBeEmpty(); + blocks.Should().OnlyContain(b => b.Success && b.BlockNumber >= 1 && b.BlockNumber <= b.BlocksTotal); + blocks.Select(b => b.FileName).Distinct().Should().BeEquivalentTo(fileStarts.Select(s => s.FileName).Distinct(), + "every executed file reports its blocks"); + } + + [Fact] + public async Task Simulate_ReportsSimulatedOutcomesWithoutARunRow() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(MigrationCommand.MigrateUp, MigrationRunMode.Simulate); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + await ctx.MigrateUpAsync(runMode: MigrationRunMode.Simulate); + ctx.AssertSuccess(true); + + recorder.Of().Should().ContainSingle().Which.MigrationRunId.Should().Be(0, "Simulate writes no run row"); + recorder.Of().Should().NotBeEmpty().And.OnlyContain(e => e.Outcome == FileOutcome.Simulated); + recorder.Of().Should().ContainSingle().Which.Success.Should().BeTrue(); + } + + [Fact] + public async Task Simulate_ReportsTheReleaseTheRunStartsFrom() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + await ctx.MigrateUpAsync("Release_1.0"); + ctx.AssertSuccess(true); + + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + await ctx.MigrateUpAsync(runMode: MigrationRunMode.Simulate); + ctx.AssertSuccess(true); + + var started = recorder.Of().Should().ContainSingle().Which; + started.MigrationRunId.Should().Be(0, "Simulate writes no run row"); + started.FromRelease.Should().Be("Release_1.0", "a run without a run row still reports where the product stands"); + } + + [Fact] + public async Task MigrateDown_WithoutRollbackFile_ReportsTheRecordAsSkipped() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario() + .RemoveRollback("Release_4.0", "02_CreateTableH.sql") + .WithRequireRollbackFile(false) + .BuildAsync(); + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + await ctx.RebuildForAsync(MigrationCommand.MigrateDown, MigrationRunMode.Migrate, "Release_3.0"); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + await ctx.MigrateDownAsync("Release_3.0"); + ctx.AssertSuccess(true); + + var ends = recorder.Of(); + var skipped = ends.Where(e => e.Outcome == FileOutcome.Skipped).Should().ContainSingle("the record without a rollback file is skipped, not rolled back").Which; + skipped.FileName.Should().Be("02_CreateTableH.sql"); + skipped.MigrationRecordId.Should().BePositive(); + skipped.ErrorMessage.Should().Contain("not found"); + ends.Where(e => e.Outcome == FileOutcome.RolledBack).Select(e => e.FileName).Should().BeEquivalentTo(["01_CreateTableG.sql", "03_SeedDataD.sql"]); + recorder.Of().Select(s => (s.FileName, s.TargetAlias, s.MigrationRecordId)) + .Should().BeEquivalentTo(ends.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), "every completion has a start, the skipped record included"); + } + + [Fact] + public async Task MigrateDown_ReportsRollbacksOfTheRecordsAboveTheTargetRelease() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + await ctx.RebuildForAsync(MigrationCommand.MigrateDown, MigrationRunMode.Migrate, "Release_2.0"); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.MigrateDownAsync("Release_2.0"); + ctx.AssertSuccess(true); + + recorder.Of().Should().ContainSingle().Which.Command.Should().Be(MigrationCommand.MigrateDown); + var ends = recorder.Of(); + ends.Should().HaveCount(result.TotalMigrations); + ends.Should().OnlyContain(e => e.Operation == MigrationOperation.MigrateDown && e.Outcome == FileOutcome.RolledBack && e.MigrationRecordId > 0); + ends.Select(e => e.ReleaseVersion).Distinct().Should().BeEquivalentTo(["Release_3.0", "Release_4.0"]); + var starts = recorder.Of(); + starts.Select(s => (s.FileName, s.TargetAlias, s.MigrationRecordId)) + .Should().BeEquivalentTo(ends.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), "every rollback completion has a start"); + starts.Should().OnlyContain(s => s.Operation == MigrationOperation.MigrateDown && s.BlocksTotal >= 1 && s.ResumeFromBlock == 0); + var blocks = recorder.Of(); + blocks.Should().NotBeEmpty("the rollback files have SQL blocks"); + blocks.Select(b => b.FileName).Distinct().Should().BeEquivalentTo(starts.Select(s => s.FileName).Distinct()); + recorder.Of().Should().ContainSingle().Which.Success.Should().BeTrue(); + } + + [Fact] + public async Task Baseline_ReportsBaselinedFilesWithoutBlocks() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(MigrationCommand.Baseline); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.BaselineAsync(); + ctx.AssertSuccess(true); + + recorder.Of().Should().ContainSingle().Which.Command.Should().Be(MigrationCommand.Baseline); + var ends = recorder.Of(); + ends.Should().HaveCount(result.BaselinedRecords, "one completion per record written") + .And.OnlyContain(e => e.Operation == MigrationOperation.Baseline && e.Outcome == FileOutcome.Baselined && e.MigrationRecordId > 0); + recorder.Of().Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)) + .Should().BeEquivalentTo(ends.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), "a start carries the record id as well"); + recorder.Of().Should().BeEmpty("baseline executes no SQL"); + recorder.Of().Should().ContainSingle().Which.TotalFiles.Should().Be(result.BaselinedRecords); + } + + [Fact] + public async Task FailingFile_ReportsTheFailedFileAndTheFailedRun() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario() + .InjectError("Release_2.0", "02_CreateTableD.sql") + .BuildAsync(); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.MigrateUpAsync(); + ctx.AssertSuccess(false); + + var failed = recorder.Of().Where(e => e.Outcome == FileOutcome.Failed).Should().ContainSingle().Which; + failed.FileName.Should().Be("02_CreateTableD.sql"); + failed.ReleaseVersion.Should().Be("Release_2.0"); + failed.ErrorMessage.Should().NotBeNullOrEmpty(); + failed.MigrationRecordId.Should().BePositive(); + recorder.Of().Where(e => e.Outcome == FileOutcome.Migrated).Should().NotBeEmpty("the files before the broken one ran"); + + var completed = recorder.Of().Should().ContainSingle().Which; + completed.Success.Should().BeFalse(); + completed.Result.Should().Be(result.Result); + completed.FailedFiles.Should().Be(result.FailedMigrations); + recorder.Events.Last().Should().BeSameAs(completed, "the run completion is the last event"); + } + + [Fact] + public async Task ObserverThatThrows_DoesNotAbortTheRun() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + ctx.UseProgressObserver(new ThrowingProgressObserver()); + + await ctx.MigrateUpAsync("Release_1.0"); + + ctx.AssertSuccess(true); + ctx.AssertRunResult(MigrationRunResult.Ok); + } + + private sealed class ThrowingProgressObserver : IMigrationProgressObserver + { + public void OnRunStarted(RunStartedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnFileStarted(FileStartedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnBlockCompleted(BlockCompletedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnFileCompleted(FileCompletedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnRunCompleted(RunCompletedEvent e) => throw new InvalidOperationException("observer failure"); + } +} diff --git a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MigrationRunMetaTests.cs b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MigrationRunMetaTests.cs index 93cb0f1..4f6eb1e 100644 --- a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MigrationRunMetaTests.cs +++ b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MigrationRunMetaTests.cs @@ -232,4 +232,23 @@ public async Task Baseline_SettingsJson_ShouldContainMaskedConnectionStrings() } } + [Fact] + public async Task FromReleaseVersion_IsTheHighestMigratedReleaseWhenTheRunStarted() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + + // The run row stamps ToReleaseVersion from the context, so the host is rebuilt for each target release. + await ctx.RebuildForAsync(MigrationCommand.MigrateUp, MigrationRunMode.Migrate, "Release_2.0"); + await ctx.MigrateUpAsync("Release_2.0"); + ctx.AssertSuccess(true); + await ctx.RebuildForAsync(MigrationCommand.MigrateUp, MigrationRunMode.Migrate); + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + ctx.AssertRunCount(2); + ctx.GetMigrationRun(1)!.FromReleaseVersion.Should().BeNull("nothing was migrated before the first run"); + ctx.AssertMigrationRun(1, new MigrationRunExpectation { ToReleaseVersion = "Release_2.0" }); + ctx.AssertMigrationRun(2, new MigrationRunExpectation { FromReleaseVersion = "Release_2.0" }); + } } diff --git a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MySqlInfoTests.cs b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MySqlInfoTests.cs index e44228e..7cddd36 100644 --- a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MySqlInfoTests.cs +++ b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MySqlInfoTests.cs @@ -326,4 +326,38 @@ public async Task GetHistory_AfterMigrateUpWithRollback_ShowsUpRunWithRolledBack history.Runs[0].SuccessfulMigrations.Should().Be(0, "every record was rolled back again"); history.Runs[0].FailedMigrations.Should().Be(0, "the failed file was rolled back too"); } + + /// + /// History and status are per environment: a run of another environment of the same product is not listed. + /// + [Fact] + public async Task GetHistory_ListsOnlyTheRunsOfTheRequestedEnvironment() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + + await using var ctx = await CreateScenario().BuildAsync(); + + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + // A second environment on the same repository: baseline writes a run row without touching the targets. + await ctx.RebuildForAsync(MigrationCommand.Baseline, MigrationRunMode.Migrate, environment: "Staging"); + await ctx.BaselineAsync(); + ctx.AssertSuccess(true); + ctx.AssertRunCount(2); + + await ctx.RebuildForAsync(MigrationCommand.Info, MigrationRunMode.Migrate, environment: "Staging"); + var staging = await ctx.GetHistoryAsync(); + staging.Environment.Should().Be("Staging"); + staging.Runs.Should().ContainSingle("the up run belongs to the Docker environment") + .Which.Operation.Should().Be(MigrationOperation.Baseline); + + await ctx.RebuildForAsync(MigrationCommand.Info, MigrationRunMode.Migrate); + var docker = await ctx.GetHistoryAsync(); + docker.Environment.Should().Be("Docker"); + docker.Runs.Should().ContainSingle("the baseline run belongs to the Staging environment") + .Which.Operation.Should().Be(MigrationOperation.MigrateUp); + docker.Runs[0].FromRelease.Should().BeNull("nothing was migrated before the first run"); + docker.Runs[0].MigrationRunId.Should().BePositive(); + } } diff --git a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MySqlMigrationRunMetaTests.cs b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MySqlMigrationRunMetaTests.cs index 5173901..0a31372 100644 --- a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MySqlMigrationRunMetaTests.cs +++ b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MySqlMigrationRunMetaTests.cs @@ -27,4 +27,23 @@ public MySqlMigrationRunMetaTests(MySqlFixture fixture) : base(fixture) { } [Fact] public async Task Baseline_SettingsJson_ShouldContainMaskedConnectionStrings() { Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); await using var ctx = await CreateScenario().BuildAsync(MigrationCommand.Baseline); await ctx.BaselineAsync(); ctx.AssertSuccess(true); var json = ctx.GetMigrationRunSettingsJson(); json.Should().NotBeNullOrEmpty("MigrationRunMeta should contain settings JSON after Baseline"); var doc = JsonDocument.Parse(json!); var repoConnStr = doc.RootElement.GetProperty("Repository").GetProperty("ConnectionString").GetString(); repoConnStr.Should().Contain(SensitiveDataMasker.MaskString, "Repository connection string should be masked in Baseline settings JSON"); var targetGroups = doc.RootElement.GetProperty("Product").GetProperty("TargetGroups"); foreach (var tg in targetGroups.EnumerateArray()) { foreach (var target in tg.GetProperty("Targets").EnumerateArray()) { var connStr = target.GetProperty("ConnectionString").GetString(); connStr.Should().Contain(SensitiveDataMasker.MaskString, $"Target [{target.GetProperty("Alias").GetString()}] connection string should be masked in Baseline settings JSON"); } } } + [Fact] + public async Task FromReleaseVersion_IsTheHighestMigratedReleaseWhenTheRunStarted() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + + // The run row stamps ToReleaseVersion from the context, so the host is rebuilt for each target release. + await ctx.RebuildForAsync(MigrationCommand.MigrateUp, MigrationRunMode.Migrate, "Release_2.0"); + await ctx.MigrateUpAsync("Release_2.0"); + ctx.AssertSuccess(true); + await ctx.RebuildForAsync(MigrationCommand.MigrateUp, MigrationRunMode.Migrate); + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + ctx.AssertRunCount(2); + ctx.GetMigrationRun(1)!.FromReleaseVersion.Should().BeNull("nothing was migrated before the first run"); + ctx.AssertMigrationRun(1, new MigrationRunExpectation { ToReleaseVersion = "Release_2.0" }); + ctx.AssertMigrationRun(2, new MigrationRunExpectation { FromReleaseVersion = "Release_2.0" }); + } } diff --git a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MySqlProgressEventsTests.cs b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MySqlProgressEventsTests.cs new file mode 100644 index 0000000..7d7ef34 --- /dev/null +++ b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/MySqlProgressEventsTests.cs @@ -0,0 +1,231 @@ +using Raycoon.RayMigrator.Core.Configuration.Enums; +using Raycoon.RayMigrator.Core.Progress; +using Raycoon.RayMigrator.Tests.Engine.Fixtures; +using Raycoon.RayMigrator.Tests.Engine.Infrastructure; + +namespace Raycoon.RayMigrator.Tests.Engine.Tests.Features; + +/// +/// The typed progress events a host observes through : one run +/// event pair per operation, one file event pair per file and target, one block event per executed block. +/// +[Collection("MySql")] +[Trait("Engine", "MySql")] +[Trait("Category", "Features")] +public class MySqlProgressEventsTests : MySqlTestBase +{ + public MySqlProgressEventsTests(MySqlFixture fixture) : base(fixture) { } + + [Fact] + public async Task MigrateUp_ReportsTheRunAndEveryFileOnEveryTarget() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.MigrateUpAsync("Release_1.0"); + ctx.AssertSuccess(true); + + recorder.Events.First().Should().BeOfType(); + recorder.Events.Last().Should().BeOfType(); + + var started = recorder.Of().Should().ContainSingle().Which; + started.Command.Should().Be(MigrationCommand.MigrateUp); + started.RunMode.Should().Be(MigrationRunMode.Migrate); + started.MigrationRunId.Should().Be(result.MigrationRunId, "the run row exists when the run is reported as started"); + started.MigrationRunId.Should().BePositive(); + started.FromRelease.Should().BeNull("nothing was migrated before the first run"); + started.TargetRelease.Should().Be("Release_1.0"); + + var completed = recorder.Of().Should().ContainSingle().Which; + completed.Success.Should().BeTrue(); + completed.Result.Should().Be(MigrationRunResult.Ok); + completed.MigrationRunId.Should().Be(result.MigrationRunId); + completed.TotalFiles.Should().Be(result.TotalMigrations); + + var fileStarts = recorder.Of(); + var fileEnds = recorder.Of(); + fileStarts.Should().HaveCount(result.TotalMigrations, "one start per file and target"); + fileEnds.Should().HaveCount(result.TotalMigrations, "one completion per file and target"); + fileEnds.Should().OnlyContain(e => e.Outcome == FileOutcome.Migrated && e.Operation == MigrationOperation.MigrateUp); + fileStarts.Should().OnlyContain(e => e.MigrationRecordId > 0 && e.ReleaseVersion == "Release_1.0" && e.BlocksTotal >= 1); + fileEnds.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)) + .Should().BeEquivalentTo(fileStarts.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), + "a completion carries the same identity as its start"); + + foreach (var end in fileEnds) + { + int startIndex = recorder.Events.IndexOf(fileStarts.Single(s => s.FileName == end.FileName && s.TargetAlias == end.TargetAlias)); + recorder.Events.IndexOf(end).Should().BeGreaterThan(startIndex, "a file completes after it started"); + } + + var blocks = recorder.Of(); + blocks.Should().NotBeEmpty(); + blocks.Should().OnlyContain(b => b.Success && b.BlockNumber >= 1 && b.BlockNumber <= b.BlocksTotal); + blocks.Select(b => b.FileName).Distinct().Should().BeEquivalentTo(fileStarts.Select(s => s.FileName).Distinct(), + "every executed file reports its blocks"); + } + + [Fact] + public async Task Simulate_ReportsSimulatedOutcomesWithoutARunRow() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(MigrationCommand.MigrateUp, MigrationRunMode.Simulate); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + await ctx.MigrateUpAsync(runMode: MigrationRunMode.Simulate); + ctx.AssertSuccess(true); + + recorder.Of().Should().ContainSingle().Which.MigrationRunId.Should().Be(0, "Simulate writes no run row"); + recorder.Of().Should().NotBeEmpty().And.OnlyContain(e => e.Outcome == FileOutcome.Simulated); + recorder.Of().Should().ContainSingle().Which.Success.Should().BeTrue(); + } + + [Fact] + public async Task Simulate_ReportsTheReleaseTheRunStartsFrom() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + await ctx.MigrateUpAsync("Release_1.0"); + ctx.AssertSuccess(true); + + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + await ctx.MigrateUpAsync(runMode: MigrationRunMode.Simulate); + ctx.AssertSuccess(true); + + var started = recorder.Of().Should().ContainSingle().Which; + started.MigrationRunId.Should().Be(0, "Simulate writes no run row"); + started.FromRelease.Should().Be("Release_1.0", "a run without a run row still reports where the product stands"); + } + + [Fact] + public async Task MigrateDown_WithoutRollbackFile_ReportsTheRecordAsSkipped() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario() + .RemoveRollback("Release_4.0", "02_CreateTableH.sql") + .WithRequireRollbackFile(false) + .BuildAsync(); + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + await ctx.RebuildForAsync(MigrationCommand.MigrateDown, MigrationRunMode.Migrate, "Release_3.0"); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + await ctx.MigrateDownAsync("Release_3.0"); + ctx.AssertSuccess(true); + + var ends = recorder.Of(); + var skipped = ends.Where(e => e.Outcome == FileOutcome.Skipped).Should().ContainSingle("the record without a rollback file is skipped, not rolled back").Which; + skipped.FileName.Should().Be("02_CreateTableH.sql"); + skipped.MigrationRecordId.Should().BePositive(); + skipped.ErrorMessage.Should().Contain("not found"); + ends.Where(e => e.Outcome == FileOutcome.RolledBack).Select(e => e.FileName).Should().BeEquivalentTo(["01_CreateTableG.sql", "03_SeedDataD.sql"]); + recorder.Of().Select(s => (s.FileName, s.TargetAlias, s.MigrationRecordId)) + .Should().BeEquivalentTo(ends.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), "every completion has a start, the skipped record included"); + } + + [Fact] + public async Task MigrateDown_ReportsRollbacksOfTheRecordsAboveTheTargetRelease() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + await ctx.RebuildForAsync(MigrationCommand.MigrateDown, MigrationRunMode.Migrate, "Release_2.0"); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.MigrateDownAsync("Release_2.0"); + ctx.AssertSuccess(true); + + recorder.Of().Should().ContainSingle().Which.Command.Should().Be(MigrationCommand.MigrateDown); + var ends = recorder.Of(); + ends.Should().HaveCount(result.TotalMigrations); + ends.Should().OnlyContain(e => e.Operation == MigrationOperation.MigrateDown && e.Outcome == FileOutcome.RolledBack && e.MigrationRecordId > 0); + ends.Select(e => e.ReleaseVersion).Distinct().Should().BeEquivalentTo(["Release_3.0", "Release_4.0"]); + var starts = recorder.Of(); + starts.Select(s => (s.FileName, s.TargetAlias, s.MigrationRecordId)) + .Should().BeEquivalentTo(ends.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), "every rollback completion has a start"); + starts.Should().OnlyContain(s => s.Operation == MigrationOperation.MigrateDown && s.BlocksTotal >= 1 && s.ResumeFromBlock == 0); + var blocks = recorder.Of(); + blocks.Should().NotBeEmpty("the rollback files have SQL blocks"); + blocks.Select(b => b.FileName).Distinct().Should().BeEquivalentTo(starts.Select(s => s.FileName).Distinct()); + recorder.Of().Should().ContainSingle().Which.Success.Should().BeTrue(); + } + + [Fact] + public async Task Baseline_ReportsBaselinedFilesWithoutBlocks() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(MigrationCommand.Baseline); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.BaselineAsync(); + ctx.AssertSuccess(true); + + recorder.Of().Should().ContainSingle().Which.Command.Should().Be(MigrationCommand.Baseline); + var ends = recorder.Of(); + ends.Should().HaveCount(result.BaselinedRecords, "one completion per record written") + .And.OnlyContain(e => e.Operation == MigrationOperation.Baseline && e.Outcome == FileOutcome.Baselined && e.MigrationRecordId > 0); + recorder.Of().Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)) + .Should().BeEquivalentTo(ends.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), "a start carries the record id as well"); + recorder.Of().Should().BeEmpty("baseline executes no SQL"); + recorder.Of().Should().ContainSingle().Which.TotalFiles.Should().Be(result.BaselinedRecords); + } + + [Fact] + public async Task FailingFile_ReportsTheFailedFileAndTheFailedRun() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario() + .InjectError("Release_2.0", "02_CreateTableD.sql") + .BuildAsync(); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.MigrateUpAsync(); + ctx.AssertSuccess(false); + + var failed = recorder.Of().Where(e => e.Outcome == FileOutcome.Failed).Should().ContainSingle().Which; + failed.FileName.Should().Be("02_CreateTableD.sql"); + failed.ReleaseVersion.Should().Be("Release_2.0"); + failed.ErrorMessage.Should().NotBeNullOrEmpty(); + failed.MigrationRecordId.Should().BePositive(); + recorder.Of().Where(e => e.Outcome == FileOutcome.Migrated).Should().NotBeEmpty("the files before the broken one ran"); + + var completed = recorder.Of().Should().ContainSingle().Which; + completed.Success.Should().BeFalse(); + completed.Result.Should().Be(result.Result); + completed.FailedFiles.Should().Be(result.FailedMigrations); + recorder.Events.Last().Should().BeSameAs(completed, "the run completion is the last event"); + } + + [Fact] + public async Task ObserverThatThrows_DoesNotAbortTheRun() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + ctx.UseProgressObserver(new ThrowingProgressObserver()); + + await ctx.MigrateUpAsync("Release_1.0"); + + ctx.AssertSuccess(true); + ctx.AssertRunResult(MigrationRunResult.Ok); + } + + private sealed class ThrowingProgressObserver : IMigrationProgressObserver + { + public void OnRunStarted(RunStartedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnFileStarted(FileStartedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnBlockCompleted(BlockCompletedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnFileCompleted(FileCompletedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnRunCompleted(RunCompletedEvent e) => throw new InvalidOperationException("observer failure"); + } +} diff --git a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/ProgressEventsTests.cs b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/ProgressEventsTests.cs new file mode 100644 index 0000000..490b788 --- /dev/null +++ b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/ProgressEventsTests.cs @@ -0,0 +1,231 @@ +using Raycoon.RayMigrator.Core.Configuration.Enums; +using Raycoon.RayMigrator.Core.Progress; +using Raycoon.RayMigrator.Tests.Engine.Fixtures; +using Raycoon.RayMigrator.Tests.Engine.Infrastructure; + +namespace Raycoon.RayMigrator.Tests.Engine.Tests.Features; + +/// +/// The typed progress events a host observes through : one run +/// event pair per operation, one file event pair per file and target, one block event per executed block. +/// +[Collection("PostgreSQL")] +[Trait("Engine", "PostgreSQL")] +[Trait("Category", "Features")] +public class ProgressEventsTests : PostgreSqlTestBase +{ + public ProgressEventsTests(PostgreSqlFixture fixture) : base(fixture) { } + + [Fact] + public async Task MigrateUp_ReportsTheRunAndEveryFileOnEveryTarget() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.MigrateUpAsync("Release_1.0"); + ctx.AssertSuccess(true); + + recorder.Events.First().Should().BeOfType(); + recorder.Events.Last().Should().BeOfType(); + + var started = recorder.Of().Should().ContainSingle().Which; + started.Command.Should().Be(MigrationCommand.MigrateUp); + started.RunMode.Should().Be(MigrationRunMode.Migrate); + started.MigrationRunId.Should().Be(result.MigrationRunId, "the run row exists when the run is reported as started"); + started.MigrationRunId.Should().BePositive(); + started.FromRelease.Should().BeNull("nothing was migrated before the first run"); + started.TargetRelease.Should().Be("Release_1.0"); + + var completed = recorder.Of().Should().ContainSingle().Which; + completed.Success.Should().BeTrue(); + completed.Result.Should().Be(MigrationRunResult.Ok); + completed.MigrationRunId.Should().Be(result.MigrationRunId); + completed.TotalFiles.Should().Be(result.TotalMigrations); + + var fileStarts = recorder.Of(); + var fileEnds = recorder.Of(); + fileStarts.Should().HaveCount(result.TotalMigrations, "one start per file and target"); + fileEnds.Should().HaveCount(result.TotalMigrations, "one completion per file and target"); + fileEnds.Should().OnlyContain(e => e.Outcome == FileOutcome.Migrated && e.Operation == MigrationOperation.MigrateUp); + fileStarts.Should().OnlyContain(e => e.MigrationRecordId > 0 && e.ReleaseVersion == "Release_1.0" && e.BlocksTotal >= 1); + fileEnds.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)) + .Should().BeEquivalentTo(fileStarts.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), + "a completion carries the same identity as its start"); + + foreach (var end in fileEnds) + { + int startIndex = recorder.Events.IndexOf(fileStarts.Single(s => s.FileName == end.FileName && s.TargetAlias == end.TargetAlias)); + recorder.Events.IndexOf(end).Should().BeGreaterThan(startIndex, "a file completes after it started"); + } + + var blocks = recorder.Of(); + blocks.Should().NotBeEmpty(); + blocks.Should().OnlyContain(b => b.Success && b.BlockNumber >= 1 && b.BlockNumber <= b.BlocksTotal); + blocks.Select(b => b.FileName).Distinct().Should().BeEquivalentTo(fileStarts.Select(s => s.FileName).Distinct(), + "every executed file reports its blocks"); + } + + [Fact] + public async Task Simulate_ReportsSimulatedOutcomesWithoutARunRow() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(MigrationCommand.MigrateUp, MigrationRunMode.Simulate); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + await ctx.MigrateUpAsync(runMode: MigrationRunMode.Simulate); + ctx.AssertSuccess(true); + + recorder.Of().Should().ContainSingle().Which.MigrationRunId.Should().Be(0, "Simulate writes no run row"); + recorder.Of().Should().NotBeEmpty().And.OnlyContain(e => e.Outcome == FileOutcome.Simulated); + recorder.Of().Should().ContainSingle().Which.Success.Should().BeTrue(); + } + + [Fact] + public async Task Simulate_ReportsTheReleaseTheRunStartsFrom() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + await ctx.MigrateUpAsync("Release_1.0"); + ctx.AssertSuccess(true); + + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + await ctx.MigrateUpAsync(runMode: MigrationRunMode.Simulate); + ctx.AssertSuccess(true); + + var started = recorder.Of().Should().ContainSingle().Which; + started.MigrationRunId.Should().Be(0, "Simulate writes no run row"); + started.FromRelease.Should().Be("Release_1.0", "a run without a run row still reports where the product stands"); + } + + [Fact] + public async Task MigrateDown_WithoutRollbackFile_ReportsTheRecordAsSkipped() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario() + .RemoveRollback("Release_4.0", "02_CreateTableH.sql") + .WithRequireRollbackFile(false) + .BuildAsync(); + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + await ctx.RebuildForAsync(MigrationCommand.MigrateDown, MigrationRunMode.Migrate, "Release_3.0"); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + await ctx.MigrateDownAsync("Release_3.0"); + ctx.AssertSuccess(true); + + var ends = recorder.Of(); + var skipped = ends.Where(e => e.Outcome == FileOutcome.Skipped).Should().ContainSingle("the record without a rollback file is skipped, not rolled back").Which; + skipped.FileName.Should().Be("02_CreateTableH.sql"); + skipped.MigrationRecordId.Should().BePositive(); + skipped.ErrorMessage.Should().Contain("not found"); + ends.Where(e => e.Outcome == FileOutcome.RolledBack).Select(e => e.FileName).Should().BeEquivalentTo(["01_CreateTableG.sql", "03_SeedDataD.sql"]); + recorder.Of().Select(s => (s.FileName, s.TargetAlias, s.MigrationRecordId)) + .Should().BeEquivalentTo(ends.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), "every completion has a start, the skipped record included"); + } + + [Fact] + public async Task MigrateDown_ReportsRollbacksOfTheRecordsAboveTheTargetRelease() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + await ctx.RebuildForAsync(MigrationCommand.MigrateDown, MigrationRunMode.Migrate, "Release_2.0"); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.MigrateDownAsync("Release_2.0"); + ctx.AssertSuccess(true); + + recorder.Of().Should().ContainSingle().Which.Command.Should().Be(MigrationCommand.MigrateDown); + var ends = recorder.Of(); + ends.Should().HaveCount(result.TotalMigrations); + ends.Should().OnlyContain(e => e.Operation == MigrationOperation.MigrateDown && e.Outcome == FileOutcome.RolledBack && e.MigrationRecordId > 0); + ends.Select(e => e.ReleaseVersion).Distinct().Should().BeEquivalentTo(["Release_3.0", "Release_4.0"]); + var starts = recorder.Of(); + starts.Select(s => (s.FileName, s.TargetAlias, s.MigrationRecordId)) + .Should().BeEquivalentTo(ends.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), "every rollback completion has a start"); + starts.Should().OnlyContain(s => s.Operation == MigrationOperation.MigrateDown && s.BlocksTotal >= 1 && s.ResumeFromBlock == 0); + var blocks = recorder.Of(); + blocks.Should().NotBeEmpty("the rollback files have SQL blocks"); + blocks.Select(b => b.FileName).Distinct().Should().BeEquivalentTo(starts.Select(s => s.FileName).Distinct()); + recorder.Of().Should().ContainSingle().Which.Success.Should().BeTrue(); + } + + [Fact] + public async Task Baseline_ReportsBaselinedFilesWithoutBlocks() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(MigrationCommand.Baseline); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.BaselineAsync(); + ctx.AssertSuccess(true); + + recorder.Of().Should().ContainSingle().Which.Command.Should().Be(MigrationCommand.Baseline); + var ends = recorder.Of(); + ends.Should().HaveCount(result.BaselinedRecords, "one completion per record written") + .And.OnlyContain(e => e.Operation == MigrationOperation.Baseline && e.Outcome == FileOutcome.Baselined && e.MigrationRecordId > 0); + recorder.Of().Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)) + .Should().BeEquivalentTo(ends.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), "a start carries the record id as well"); + recorder.Of().Should().BeEmpty("baseline executes no SQL"); + recorder.Of().Should().ContainSingle().Which.TotalFiles.Should().Be(result.BaselinedRecords); + } + + [Fact] + public async Task FailingFile_ReportsTheFailedFileAndTheFailedRun() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario() + .InjectError("Release_2.0", "02_CreateTableD.sql") + .BuildAsync(); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.MigrateUpAsync(); + ctx.AssertSuccess(false); + + var failed = recorder.Of().Where(e => e.Outcome == FileOutcome.Failed).Should().ContainSingle().Which; + failed.FileName.Should().Be("02_CreateTableD.sql"); + failed.ReleaseVersion.Should().Be("Release_2.0"); + failed.ErrorMessage.Should().NotBeNullOrEmpty(); + failed.MigrationRecordId.Should().BePositive(); + recorder.Of().Where(e => e.Outcome == FileOutcome.Migrated).Should().NotBeEmpty("the files before the broken one ran"); + + var completed = recorder.Of().Should().ContainSingle().Which; + completed.Success.Should().BeFalse(); + completed.Result.Should().Be(result.Result); + completed.FailedFiles.Should().Be(result.FailedMigrations); + recorder.Events.Last().Should().BeSameAs(completed, "the run completion is the last event"); + } + + [Fact] + public async Task ObserverThatThrows_DoesNotAbortTheRun() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + ctx.UseProgressObserver(new ThrowingProgressObserver()); + + await ctx.MigrateUpAsync("Release_1.0"); + + ctx.AssertSuccess(true); + ctx.AssertRunResult(MigrationRunResult.Ok); + } + + private sealed class ThrowingProgressObserver : IMigrationProgressObserver + { + public void OnRunStarted(RunStartedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnFileStarted(FileStartedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnBlockCompleted(BlockCompletedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnFileCompleted(FileCompletedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnRunCompleted(RunCompletedEvent e) => throw new InvalidOperationException("observer failure"); + } +} diff --git a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqlServerInfoTests.cs b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqlServerInfoTests.cs index 6d85ffb..4292f20 100644 --- a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqlServerInfoTests.cs +++ b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqlServerInfoTests.cs @@ -326,4 +326,38 @@ public async Task GetHistory_AfterMigrateUpWithRollback_ShowsUpRunWithRolledBack history.Runs[0].SuccessfulMigrations.Should().Be(0, "every record was rolled back again"); history.Runs[0].FailedMigrations.Should().Be(0, "the failed file was rolled back too"); } + + /// + /// History and status are per environment: a run of another environment of the same product is not listed. + /// + [Fact] + public async Task GetHistory_ListsOnlyTheRunsOfTheRequestedEnvironment() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + + await using var ctx = await CreateScenario().BuildAsync(); + + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + // A second environment on the same repository: baseline writes a run row without touching the targets. + await ctx.RebuildForAsync(MigrationCommand.Baseline, MigrationRunMode.Migrate, environment: "Staging"); + await ctx.BaselineAsync(); + ctx.AssertSuccess(true); + ctx.AssertRunCount(2); + + await ctx.RebuildForAsync(MigrationCommand.Info, MigrationRunMode.Migrate, environment: "Staging"); + var staging = await ctx.GetHistoryAsync(); + staging.Environment.Should().Be("Staging"); + staging.Runs.Should().ContainSingle("the up run belongs to the Docker environment") + .Which.Operation.Should().Be(MigrationOperation.Baseline); + + await ctx.RebuildForAsync(MigrationCommand.Info, MigrationRunMode.Migrate); + var docker = await ctx.GetHistoryAsync(); + docker.Environment.Should().Be("Docker"); + docker.Runs.Should().ContainSingle("the baseline run belongs to the Staging environment") + .Which.Operation.Should().Be(MigrationOperation.MigrateUp); + docker.Runs[0].FromRelease.Should().BeNull("nothing was migrated before the first run"); + docker.Runs[0].MigrationRunId.Should().BePositive(); + } } diff --git a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqlServerMigrationRunMetaTests.cs b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqlServerMigrationRunMetaTests.cs index 7306ad2..535bb17 100644 --- a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqlServerMigrationRunMetaTests.cs +++ b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqlServerMigrationRunMetaTests.cs @@ -232,4 +232,23 @@ public async Task Baseline_SettingsJson_ShouldContainMaskedConnectionStrings() } } + [Fact] + public async Task FromReleaseVersion_IsTheHighestMigratedReleaseWhenTheRunStarted() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + + // The run row stamps ToReleaseVersion from the context, so the host is rebuilt for each target release. + await ctx.RebuildForAsync(MigrationCommand.MigrateUp, MigrationRunMode.Migrate, "Release_2.0"); + await ctx.MigrateUpAsync("Release_2.0"); + ctx.AssertSuccess(true); + await ctx.RebuildForAsync(MigrationCommand.MigrateUp, MigrationRunMode.Migrate); + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + ctx.AssertRunCount(2); + ctx.GetMigrationRun(1)!.FromReleaseVersion.Should().BeNull("nothing was migrated before the first run"); + ctx.AssertMigrationRun(1, new MigrationRunExpectation { ToReleaseVersion = "Release_2.0" }); + ctx.AssertMigrationRun(2, new MigrationRunExpectation { FromReleaseVersion = "Release_2.0" }); + } } diff --git a/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqlServerProgressEventsTests.cs b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqlServerProgressEventsTests.cs new file mode 100644 index 0000000..ebff4c3 --- /dev/null +++ b/Raycoon.RayMigrator.Tests.Engine/Tests/Features/SqlServerProgressEventsTests.cs @@ -0,0 +1,231 @@ +using Raycoon.RayMigrator.Core.Configuration.Enums; +using Raycoon.RayMigrator.Core.Progress; +using Raycoon.RayMigrator.Tests.Engine.Fixtures; +using Raycoon.RayMigrator.Tests.Engine.Infrastructure; + +namespace Raycoon.RayMigrator.Tests.Engine.Tests.Features; + +/// +/// The typed progress events a host observes through : one run +/// event pair per operation, one file event pair per file and target, one block event per executed block. +/// +[Collection("SqlServer")] +[Trait("Engine", "SqlServer")] +[Trait("Category", "Features")] +public class SqlServerProgressEventsTests : SqlServerTestBase +{ + public SqlServerProgressEventsTests(SqlServerFixture fixture) : base(fixture) { } + + [Fact] + public async Task MigrateUp_ReportsTheRunAndEveryFileOnEveryTarget() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.MigrateUpAsync("Release_1.0"); + ctx.AssertSuccess(true); + + recorder.Events.First().Should().BeOfType(); + recorder.Events.Last().Should().BeOfType(); + + var started = recorder.Of().Should().ContainSingle().Which; + started.Command.Should().Be(MigrationCommand.MigrateUp); + started.RunMode.Should().Be(MigrationRunMode.Migrate); + started.MigrationRunId.Should().Be(result.MigrationRunId, "the run row exists when the run is reported as started"); + started.MigrationRunId.Should().BePositive(); + started.FromRelease.Should().BeNull("nothing was migrated before the first run"); + started.TargetRelease.Should().Be("Release_1.0"); + + var completed = recorder.Of().Should().ContainSingle().Which; + completed.Success.Should().BeTrue(); + completed.Result.Should().Be(MigrationRunResult.Ok); + completed.MigrationRunId.Should().Be(result.MigrationRunId); + completed.TotalFiles.Should().Be(result.TotalMigrations); + + var fileStarts = recorder.Of(); + var fileEnds = recorder.Of(); + fileStarts.Should().HaveCount(result.TotalMigrations, "one start per file and target"); + fileEnds.Should().HaveCount(result.TotalMigrations, "one completion per file and target"); + fileEnds.Should().OnlyContain(e => e.Outcome == FileOutcome.Migrated && e.Operation == MigrationOperation.MigrateUp); + fileStarts.Should().OnlyContain(e => e.MigrationRecordId > 0 && e.ReleaseVersion == "Release_1.0" && e.BlocksTotal >= 1); + fileEnds.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)) + .Should().BeEquivalentTo(fileStarts.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), + "a completion carries the same identity as its start"); + + foreach (var end in fileEnds) + { + int startIndex = recorder.Events.IndexOf(fileStarts.Single(s => s.FileName == end.FileName && s.TargetAlias == end.TargetAlias)); + recorder.Events.IndexOf(end).Should().BeGreaterThan(startIndex, "a file completes after it started"); + } + + var blocks = recorder.Of(); + blocks.Should().NotBeEmpty(); + blocks.Should().OnlyContain(b => b.Success && b.BlockNumber >= 1 && b.BlockNumber <= b.BlocksTotal); + blocks.Select(b => b.FileName).Distinct().Should().BeEquivalentTo(fileStarts.Select(s => s.FileName).Distinct(), + "every executed file reports its blocks"); + } + + [Fact] + public async Task Simulate_ReportsSimulatedOutcomesWithoutARunRow() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(MigrationCommand.MigrateUp, MigrationRunMode.Simulate); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + await ctx.MigrateUpAsync(runMode: MigrationRunMode.Simulate); + ctx.AssertSuccess(true); + + recorder.Of().Should().ContainSingle().Which.MigrationRunId.Should().Be(0, "Simulate writes no run row"); + recorder.Of().Should().NotBeEmpty().And.OnlyContain(e => e.Outcome == FileOutcome.Simulated); + recorder.Of().Should().ContainSingle().Which.Success.Should().BeTrue(); + } + + [Fact] + public async Task Simulate_ReportsTheReleaseTheRunStartsFrom() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + await ctx.MigrateUpAsync("Release_1.0"); + ctx.AssertSuccess(true); + + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + await ctx.MigrateUpAsync(runMode: MigrationRunMode.Simulate); + ctx.AssertSuccess(true); + + var started = recorder.Of().Should().ContainSingle().Which; + started.MigrationRunId.Should().Be(0, "Simulate writes no run row"); + started.FromRelease.Should().Be("Release_1.0", "a run without a run row still reports where the product stands"); + } + + [Fact] + public async Task MigrateDown_WithoutRollbackFile_ReportsTheRecordAsSkipped() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario() + .RemoveRollback("Release_4.0", "02_CreateTableH.sql") + .WithRequireRollbackFile(false) + .BuildAsync(); + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + await ctx.RebuildForAsync(MigrationCommand.MigrateDown, MigrationRunMode.Migrate, "Release_3.0"); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + await ctx.MigrateDownAsync("Release_3.0"); + ctx.AssertSuccess(true); + + var ends = recorder.Of(); + var skipped = ends.Where(e => e.Outcome == FileOutcome.Skipped).Should().ContainSingle("the record without a rollback file is skipped, not rolled back").Which; + skipped.FileName.Should().Be("02_CreateTableH.sql"); + skipped.MigrationRecordId.Should().BePositive(); + skipped.ErrorMessage.Should().Contain("not found"); + ends.Where(e => e.Outcome == FileOutcome.RolledBack).Select(e => e.FileName).Should().BeEquivalentTo(["01_CreateTableG.sql", "03_SeedDataD.sql"]); + recorder.Of().Select(s => (s.FileName, s.TargetAlias, s.MigrationRecordId)) + .Should().BeEquivalentTo(ends.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), "every completion has a start, the skipped record included"); + } + + [Fact] + public async Task MigrateDown_ReportsRollbacksOfTheRecordsAboveTheTargetRelease() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + await ctx.MigrateUpAsync(); + ctx.AssertSuccess(true); + + await ctx.RebuildForAsync(MigrationCommand.MigrateDown, MigrationRunMode.Migrate, "Release_2.0"); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.MigrateDownAsync("Release_2.0"); + ctx.AssertSuccess(true); + + recorder.Of().Should().ContainSingle().Which.Command.Should().Be(MigrationCommand.MigrateDown); + var ends = recorder.Of(); + ends.Should().HaveCount(result.TotalMigrations); + ends.Should().OnlyContain(e => e.Operation == MigrationOperation.MigrateDown && e.Outcome == FileOutcome.RolledBack && e.MigrationRecordId > 0); + ends.Select(e => e.ReleaseVersion).Distinct().Should().BeEquivalentTo(["Release_3.0", "Release_4.0"]); + var starts = recorder.Of(); + starts.Select(s => (s.FileName, s.TargetAlias, s.MigrationRecordId)) + .Should().BeEquivalentTo(ends.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), "every rollback completion has a start"); + starts.Should().OnlyContain(s => s.Operation == MigrationOperation.MigrateDown && s.BlocksTotal >= 1 && s.ResumeFromBlock == 0); + var blocks = recorder.Of(); + blocks.Should().NotBeEmpty("the rollback files have SQL blocks"); + blocks.Select(b => b.FileName).Distinct().Should().BeEquivalentTo(starts.Select(s => s.FileName).Distinct()); + recorder.Of().Should().ContainSingle().Which.Success.Should().BeTrue(); + } + + [Fact] + public async Task Baseline_ReportsBaselinedFilesWithoutBlocks() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(MigrationCommand.Baseline); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.BaselineAsync(); + ctx.AssertSuccess(true); + + recorder.Of().Should().ContainSingle().Which.Command.Should().Be(MigrationCommand.Baseline); + var ends = recorder.Of(); + ends.Should().HaveCount(result.BaselinedRecords, "one completion per record written") + .And.OnlyContain(e => e.Operation == MigrationOperation.Baseline && e.Outcome == FileOutcome.Baselined && e.MigrationRecordId > 0); + recorder.Of().Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)) + .Should().BeEquivalentTo(ends.Select(e => (e.FileName, e.TargetAlias, e.MigrationRecordId)), "a start carries the record id as well"); + recorder.Of().Should().BeEmpty("baseline executes no SQL"); + recorder.Of().Should().ContainSingle().Which.TotalFiles.Should().Be(result.BaselinedRecords); + } + + [Fact] + public async Task FailingFile_ReportsTheFailedFileAndTheFailedRun() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario() + .InjectError("Release_2.0", "02_CreateTableD.sql") + .BuildAsync(); + var recorder = new RecordingProgressObserver(); + ctx.UseProgressObserver(recorder); + + var result = await ctx.MigrateUpAsync(); + ctx.AssertSuccess(false); + + var failed = recorder.Of().Where(e => e.Outcome == FileOutcome.Failed).Should().ContainSingle().Which; + failed.FileName.Should().Be("02_CreateTableD.sql"); + failed.ReleaseVersion.Should().Be("Release_2.0"); + failed.ErrorMessage.Should().NotBeNullOrEmpty(); + failed.MigrationRecordId.Should().BePositive(); + recorder.Of().Where(e => e.Outcome == FileOutcome.Migrated).Should().NotBeEmpty("the files before the broken one ran"); + + var completed = recorder.Of().Should().ContainSingle().Which; + completed.Success.Should().BeFalse(); + completed.Result.Should().Be(result.Result); + completed.FailedFiles.Should().Be(result.FailedMigrations); + recorder.Events.Last().Should().BeSameAs(completed, "the run completion is the last event"); + } + + [Fact] + public async Task ObserverThatThrows_DoesNotAbortTheRun() + { + Assert.SkipUnless(Fixture.IsDatabaseAvailable, "Docker not available"); + await using var ctx = await CreateScenario().BuildAsync(); + ctx.UseProgressObserver(new ThrowingProgressObserver()); + + await ctx.MigrateUpAsync("Release_1.0"); + + ctx.AssertSuccess(true); + ctx.AssertRunResult(MigrationRunResult.Ok); + } + + private sealed class ThrowingProgressObserver : IMigrationProgressObserver + { + public void OnRunStarted(RunStartedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnFileStarted(FileStartedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnBlockCompleted(BlockCompletedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnFileCompleted(FileCompletedEvent e) => throw new InvalidOperationException("observer failure"); + public void OnRunCompleted(RunCompletedEvent e) => throw new InvalidOperationException("observer failure"); + } +} From cdb442682c4cd6a2d250a1b03a7f87b02baeb99b Mon Sep 17 00:00:00 2001 From: danielpavic Date: Mon, 12 Oct 2026 23:56:05 +0200 Subject: [PATCH 14/14] Sync the docs with the TemplateCache constructor Six documents still showed the constructor that took the options and a validateConfiguration flag. They now describe the constructor without options and the explicit ValidateConfigurationAgainstTemplateCache call per configuration. --- Docs/01-architecture/dependency-injection.md | 14 +++----------- Docs/01-architecture/design-decisions.md | 9 ++++----- Docs/01-architecture/patterns.md | 7 +++---- Docs/04-service-layer/template-executor.md | 6 +++--- Docs/09-extending/template-customization.md | 5 ++--- Docs/arc42/09-Architecture-Decisions.md | 2 +- 6 files changed, 16 insertions(+), 27 deletions(-) diff --git a/Docs/01-architecture/dependency-injection.md b/Docs/01-architecture/dependency-injection.md index c1c8f43..090dd32 100644 --- a/Docs/01-architecture/dependency-injection.md +++ b/Docs/01-architecture/dependency-injection.md @@ -364,30 +364,22 @@ public class TemplateExecutor } ``` -### TemplateCache (Decoupled from MigrationContext) +### TemplateCache (Decoupled from MigrationContext and from the options) ```csharp public class TemplateCache { - private readonly RayMigratorOptions? _options; private readonly bool _revealSensitiveData; private readonly ILogger _logger; - public TemplateCache( - IOptions? options, - bool revealSensitiveData, - ILogger logger, - bool validateConfiguration = true) + public TemplateCache(bool revealSensitiveData, ILogger logger) { - _options = options?.Value; _revealSensitiveData = revealSensitiveData; _logger = logger; Initialize(); - if (validateConfiguration && _options != null) - ValidateConfigurationAgainstTemplateCache(_options); } - /// Can be called on-demand when Products/Repository config becomes available + /// Called by the host for every configuration it runs; the CLI pipeline calls it once after the options are bound public void ValidateConfigurationAgainstTemplateCache(RayMigratorOptions options) { ... } /// Gets all available DAL names (e.g. "SqlServer", "PostgreSQL", "MariaDb", "MySql") diff --git a/Docs/01-architecture/design-decisions.md b/Docs/01-architecture/design-decisions.md index 1ec3aaa..3726ef3 100644 --- a/Docs/01-architecture/design-decisions.md +++ b/Docs/01-architecture/design-decisions.md @@ -79,14 +79,13 @@ All four files are searched in the directory specified by `--config-dir` (or the - `{ENV:VariableName}` - Environment variables - `@ParameterName` - SQL parameters (parameterized queries) -### TemplateCache with Deferred Validation +### TemplateCache without options -**Decision**: `TemplateCache` accepts a `validateConfiguration` parameter (default `true`). When set to `false`, template loading occurs without validating that configured `DatabaseType` values have matching templates. +**Decision**: `TemplateCache` takes no options. It loads every template at construction, and the host calls `ValidateConfigurationAgainstTemplateCache(options)` for each configuration it runs. **Rationale**: -- **Flexibility**: When Products and Repository configuration are loaded from an alternative source, they may not be available at `TemplateCache` construction time. The cache loads all available templates at startup but defers validation until `ValidateConfigurationAgainstTemplateCache(options)` is called explicitly after configuration is loaded. -- **Reusability**: The same `TemplateCache` class works for different configuration modes without branching logic. -- **Public validation method**: `ValidateConfigurationAgainstTemplateCache(options)` and `GetAvailableDatabaseTypes()` are public, allowing external callers to validate on demand. +- **One cache, many configurations**: A host that runs one configuration per request (RayMigrator Studio) builds the cache once per process; a constructor bound to one options object would tie the cache to the first configuration. +- **Explicit validation**: The CLI pipeline validates once after the options are bound; a host validates whenever a configuration enters. `GetAvailableDatabaseTypes()` stays public for diagnostics. ## MigrationContext Pattern diff --git a/Docs/01-architecture/patterns.md b/Docs/01-architecture/patterns.md index f96e06d..40b9024 100644 --- a/Docs/01-architecture/patterns.md +++ b/Docs/01-architecture/patterns.md @@ -178,15 +178,14 @@ public class TemplateExecutor } ``` -The `TemplateCache` (in `Raycoon.RayMigrator.Core.Templates`) loads and caches all SQL templates at startup. It supports deferred validation via the `validateConfiguration` parameter, which is set to `false` when Products configuration is not yet available at cache initialization: +The `TemplateCache` (in `Raycoon.RayMigrator.Core.Templates`) loads and caches all SQL templates once per process. It takes no options: the host validates each configuration it runs against the cache explicitly, so one cache serves many configurations: ```csharp public class TemplateCache { - public TemplateCache(IOptions? options, bool revealSensitiveData, - ILogger logger, bool validateConfiguration = true) { /* ... */ } + public TemplateCache(bool revealSensitiveData, ILogger logger) { /* ... */ } - // Called on-demand when configuration becomes available + // Called by the host for every configuration it runs (the CLI pipeline once, after the options are bound) public void ValidateConfigurationAgainstTemplateCache(RayMigratorOptions options) { /* ... */ } // Returns all loaded DAL types (e.g. "SqlServer", "PostgreSQL", "MariaDb", "MySql", "Sqlite") diff --git a/Docs/04-service-layer/template-executor.md b/Docs/04-service-layer/template-executor.md index 586109c..cf268d2 100644 --- a/Docs/04-service-layer/template-executor.md +++ b/Docs/04-service-layer/template-executor.md @@ -274,12 +274,12 @@ The `TemplateCache` is eagerly initialized at construction time (not lazy-loaded public TemplateCache(bool revealSensitiveData, ILogger logger) ``` -- `options` is nullable. When null (or when `validateConfiguration` is false), configuration validation is deferred. This supports ManagedLocal mode where Products/Repository config is not yet known at startup. -- When `validateConfiguration` is true and `options` is non-null, the constructor calls `ValidateConfigurationAgainstTemplateCache(options)` to ensure all configured `DatabaseType` values have matching DAL templates. +- The constructor loads the templates of every DAL folder; it knows no configuration. +- The host calls `ValidateConfigurationAgainstTemplateCache(options)` for each configuration it runs (the CLI pipeline once, after the options are bound) to ensure all configured `DatabaseType` values have matching DAL templates. **Key public methods**: -- `ValidateConfigurationAgainstTemplateCache(RayMigratorOptions options)` -- Validates that the Repository and all TargetGroup `DatabaseType` values have matching templates in the cache. Can be called on-demand when configuration becomes available (e.g., loaded from Admin-DB). +- `ValidateConfigurationAgainstTemplateCache(RayMigratorOptions options)` -- Validates that the Repository and all TargetGroup `DatabaseType` values have matching templates in the cache. The CLI pipeline calls it once after the options are bound; a host calls it for every configuration it runs. - `GetAvailableDatabaseTypes()` -- Returns the list of all discovered DAL database type names (loaded from `DataAccessLayers/` subdirectories). - `GetRepositoryTemplate(TemplateType, T) where T : RepositoryOptions` -- Returns a template with `{CFG:*}` placeholders replaced using the passed `RepositoryOptions` property class. The database type is inferred from `T.DatabaseType`. Throws `ConfigurationValidationException` if unreplaced `{CFG:*}` placeholders remain. - `GetTemplate(string databaseType, TemplateType, T)` -- Same as above but for non-repository templates, requiring an explicit `databaseType` parameter. diff --git a/Docs/09-extending/template-customization.md b/Docs/09-extending/template-customization.md index 2dfd92b..1058d78 100644 --- a/Docs/09-extending/template-customization.md +++ b/Docs/09-extending/template-customization.md @@ -520,8 +520,7 @@ public class TemplateCache // {ENV:*} placeholders are replaced during Initialize() (once at startup) // {CFG:*} placeholders are replaced per call in GetTemplate/GetRepositoryTemplate - public TemplateCache(IOptions? options, bool revealSensitiveData, - ILogger logger, bool validateConfiguration = true) { ... } + public TemplateCache(bool revealSensitiveData, ILogger logger) { ... } // Template retrieval (all replace {CFG:*} placeholders via reflection) public Template GetTemplate(string databaseType, TemplateType templateType, T propertyClass) { ... } @@ -535,7 +534,7 @@ public class TemplateCache } ``` -The `validateConfiguration` constructor parameter (default `true`) controls whether Products/Repository configuration is validated against loaded templates at construction time. Set to `false` when Products/Repository config is not yet known at construction time. +The constructor loads the templates only. `ValidateConfigurationAgainstTemplateCache(options)` checks that the Repository, the DatabaseLogging and every TargetGroup `DatabaseType` of a configuration has templates; the CLI pipeline calls it once after the options are bound, a host for every configuration it runs. ## Testing Custom Templates diff --git a/Docs/arc42/09-Architecture-Decisions.md b/Docs/arc42/09-Architecture-Decisions.md index 1176b2e..2643e84 100644 --- a/Docs/arc42/09-Architecture-Decisions.md +++ b/Docs/arc42/09-Architecture-Decisions.md @@ -125,7 +125,7 @@ a rebuild. `TemplateType` (21 besides `Undefined`), loaded by `TemplateCache` at startup and executed by `TemplateExecutor`. Every template returns one scalar `ResultCode,ResultMessage`; negative codes map to the `TemplateResultCode` catalog and raise `TemplateResultException`. Placeholders are `{ENV:*}` (once, at load), `{CFG:*}` (per call, `SchemaName` and `TableBaseName` only) -and `@Parameter` bindings; `TemplateCache` checks completeness at startup; `validateConfiguration = false` only skips the check that every configured `DatabaseType` has a template folder. +and `@Parameter` bindings; `TemplateCache` checks completeness at startup, and `ValidateConfigurationAgainstTemplateCache` checks per configuration that every configured `DatabaseType` has a template folder. **Consequences.** Positive: no ORM, engine specific locking primitives (`UPDLOCK, HOLDLOCK`, `pg_advisory_xact_lock`, `GET_LOCK`) live in SQL where they belong, and a DBA can customize a template. Negative: 21 templates times five engines to keep in sync, a