diff --git a/src/TALXIS.CLI.Platform.Xrm/CmtImportRunner.cs b/src/TALXIS.CLI.Platform.Xrm/CmtImportRunner.cs index fd5d4656..900588bb 100644 --- a/src/TALXIS.CLI.Platform.Xrm/CmtImportRunner.cs +++ b/src/TALXIS.CLI.Platform.Xrm/CmtImportRunner.cs @@ -40,6 +40,23 @@ static CmtImportRunner() /// private int _failedStageCount; + /// + /// Guards the one-shot LookupKeys pre-population so it fires exactly once + /// when CMT fires the first "Processing Entity:" progress event (which is + /// after ImportCommonMethods.dataEntities is populated and + /// ClearCrossReferanceList() has already run). + /// + private int _lookupKeysPrepopulated; + + /// + /// Runtime type of the ImportCrmDataHandler (set during RunInternalAsync). + /// Used by to navigate to + /// the correct runtime-loaded DataMigCommon assembly, which may differ from + /// the net462 compile-time reference if the assembly resolver returned a + /// different instance. + /// + private Type? _handlerRuntimeType; + public CmtImportRunner() { _assemblyMap = new Dictionary( @@ -191,6 +208,10 @@ private async Task RunInternalAsync( ConfigurationManager.AppSettings["ExportFiles"] = "true"; // 4. Wire progress event handlers. + // Capture the RUNTIME type so PrePopulateLookupKeysFromPackage can + // navigate to the correct DataMigCommon assembly instance (which may + // differ from the net462 compile-time reference). + _handlerRuntimeType = handler.GetType(); handler.AddNewProgressItem += OnAddNewProgressItem; handler.UpdateProgressItem += OnUpdateProgressItem; @@ -348,6 +369,19 @@ private void TryWireCmtConsoleLogging(ImportCrmDataHandler handler, bool verbose private void OnAddNewProgressItem(object? sender, ProgressItemEventArgs e) { + // "Processing Entity: " is the first AddNewProgressItem fired by + // ImportCrmEntityActions.BeginEntityImport() — it fires AFTER + // ImportDataToCrm() has called ImportCommonMethods.ClearCrossReferanceList() + // (which clears LookupKeys), so this is the correct point to pre-seed the + // cache. Hooking on "Schema Validation Complete" fires too early — CMT wipes + // LookupKeys via ClearCrossReferanceList() after that event returns. + string message = e.progressItem?.ItemText ?? string.Empty; + if (message.StartsWith("Processing Entity:", StringComparison.OrdinalIgnoreCase) + && Interlocked.CompareExchange(ref _lookupKeysPrepopulated, 1, 0) == 0) + { + CmtLookupKeysPrepopulator.Prepopulate(_handlerRuntimeType, _logger); + } + OnUpdateProgressItem(sender, e); } @@ -357,6 +391,7 @@ private void OnUpdateProgressItem(object? sender, ProgressItemEventArgs e) return; string message = e.progressItem.ItemText ?? string.Empty; + switch (e.progressItem.ItemStatus) { case ProgressItemStatus.Complete: @@ -427,4 +462,5 @@ private void RegisterExtractedDirectoryForProbing(string directory) _unresolvedAssemblies.Add(key); return null; } + } diff --git a/src/TALXIS.CLI.Platform.Xrm/CmtLookupKeysPrepopulator.cs b/src/TALXIS.CLI.Platform.Xrm/CmtLookupKeysPrepopulator.cs new file mode 100644 index 00000000..3c43d1d4 --- /dev/null +++ b/src/TALXIS.CLI.Platform.Xrm/CmtLookupKeysPrepopulator.cs @@ -0,0 +1,284 @@ +using System.Reflection; +using Microsoft.Extensions.Logging; + +namespace TALXIS.CLI.Platform.Xrm; + +/// +/// Pre-seeds CMT's internal lookup-resolution state for package-internal +/// references, eliminating redundant "LOOKUP TO CRM" server round-trips +/// during import. +/// +/// +/// Root cause: CMT's FindEntity() +/// (ImportCrmEntityActions.cs) falls back to a live server query +/// whenever a referenced record's in-memory newId is empty. +/// newId is only populated once the batch response callback for +/// that specific record's create/upsert has fired. With +/// --batch-mode and --connection-count > 1, CMT dispatches +/// an entity's batches asynchronously and can begin preprocessing the next +/// (dependent) entity before every parent batch has actually received its +/// response — so a perfectly valid, already-created parent record still +/// looks "unresolved" locally, triggering a costly live lookup that almost +/// always just confirms what CMT could already have known. This is a local +/// cache-staleness / async race inherent to CMT's batch+multi-connection +/// architecture, not a genuine missing-dependency or ordering problem. +/// +/// +/// +/// Fix: because CMT always preserves source GUIDs (an upserted +/// record's target Id is set to the source record's GUID), the +/// target GUID is always equal to the source GUID for package-internal +/// references. We can therefore pre-seed both of FindEntity()'s +/// lookup paths — the LookupKeys cache and each record's own +/// newId field — before any batch is ever dispatched, removing the +/// dependency on batch-response timing entirely. This does not affect +/// external lookups (entities not present in the package); those still +/// resolve via the normal server call path. +/// +/// +/// +/// Accesses CMT internals via reflection because +/// Microsoft.Xrm.Tooling.Dmt.DataMigCommon is a net462 legacy +/// assembly that is patched and loaded at runtime — it is not directly +/// referenceable from the net10 host at compile time. +/// +/// +internal static class CmtLookupKeysPrepopulator +{ + private const string ImportCommonMethodsTypeName = + "Microsoft.Xrm.Tooling.Dmt.DataMigCommon.DataInteraction.ImportCommonMethods"; + + /// + /// Result of a pre-population run, for logging/telemetry by the caller. + /// + public readonly record struct Result(int RecordCount, int EntityCount); + + /// + /// Pre-seeds the LookupKeys cache and newId fields for every record in + /// the already-loaded package. Safe to call multiple times; each call + /// re-scans the current in-memory package data. + /// + /// + /// Runtime type of the ImportCrmDataHandler instance in use. Used to + /// navigate to the correct runtime-loaded DataMigCommon assembly, which + /// may differ from the net462 compile-time reference if the assembly + /// resolver returned a different instance (there can be two loaded + /// copies of DataMigCommon with independent static state). + /// + /// Logger for diagnostics. + /// + /// The number of records/entities updated, or null if + /// pre-population could not run (e.g. package not loaded yet). + /// + public static Result? Prepopulate(Type? handlerRuntimeType, ILogger logger) + { + try + { + Type? importCommonType = ResolveImportCommonMethodsType(handlerRuntimeType); + if (importCommonType == null) + { + logger.LogInformation( + "LookupKeys pre-population skipped — ImportCommonMethods type not found in loaded assemblies"); + return null; + } + + logger.LogInformation( + "LookupKeys pre-population: using type from assembly {Assembly}", + importCommonType.Assembly.FullName); + + if (!TryGetLookupKeysTryAdd(importCommonType, logger, out object? lookupKeys, out MethodInfo? tryAdd)) + return null; + + if (!TryGetDataEntities(importCommonType, logger, out System.Collections.IEnumerable? entityArray)) + return null; + + Result result = PrepopulateRecords(entityArray!, lookupKeys!, tryAdd!); + + logger.LogInformation( + "Pre-populated LookupKeys cache and set newId for {Count} records across {EntityCount} entities — " + + "eliminates server lookup calls for internal package references", + result.RecordCount, result.EntityCount); + + return result; + } + catch (Exception ex) + { + logger.LogInformation(ex, + "LookupKeys pre-population failed — import will proceed with standard CMT lookup behavior"); + return null; + } + } + + /// + /// Navigates to the RUNTIME-loaded DataMigCommon assembly via the + /// handler's type. AppDomain.GetAssemblies() may contain both the + /// compile-time net462 copy and the runtime-loaded copy; they carry + /// separate static state, so we must find the one CMT actually uses. + /// + private static Type? ResolveImportCommonMethodsType(Type? handlerRuntimeType) + { + if (handlerRuntimeType != null) + { + foreach (AssemblyName asmRef in handlerRuntimeType.Assembly.GetReferencedAssemblies()) + { + if (asmRef.Name?.Contains("DataMigCommon", StringComparison.OrdinalIgnoreCase) != true) + continue; + + Assembly? loaded = AppDomain.CurrentDomain.GetAssemblies() + .FirstOrDefault(a => string.Equals(a.GetName().Name, asmRef.Name, StringComparison.OrdinalIgnoreCase)); + if (loaded == null) continue; + + Type? candidate = loaded.GetType(ImportCommonMethodsTypeName); + if (candidate != null) return candidate; + } + } + + // Fallback: scan all loaded assemblies (less precise but still useful). + return AppDomain.CurrentDomain.GetAssemblies() + .Where(a => a.GetName().Name?.Contains("DataMigCommon", StringComparison.OrdinalIgnoreCase) == true) + .Select(a => a.GetType(ImportCommonMethodsTypeName)) + .FirstOrDefault(t => t != null); + } + + /// + /// Retrieves the static LookupKeys ConcurrentDictionary<string, Guid> + /// and its TryAdd(string, Guid) method. + /// + private static bool TryGetLookupKeysTryAdd( + Type importCommonType, + ILogger logger, + out object? lookupKeys, + out MethodInfo? tryAdd) + { + lookupKeys = null; + tryAdd = null; + + FieldInfo? lookupKeysField = importCommonType.GetField( + "LookupKeys", BindingFlags.Public | BindingFlags.Static); + lookupKeys = lookupKeysField?.GetValue(null); + if (lookupKeys == null) + { + logger.LogInformation("LookupKeys pre-population skipped — LookupKeys field not found or null"); + return false; + } + + tryAdd = lookupKeys.GetType().GetMethod("TryAdd", [typeof(string), typeof(Guid)]); + if (tryAdd == null) + { + logger.LogInformation("LookupKeys pre-population skipped — TryAdd method not found on LookupKeys"); + return false; + } + + return true; + } + + /// + /// Retrieves the static dataEntities property and its entity array. + /// + private static bool TryGetDataEntities( + Type importCommonType, + ILogger logger, + out System.Collections.IEnumerable? entityArray) + { + entityArray = null; + + PropertyInfo? dataEntitiesProp = importCommonType.GetProperty( + "dataEntities", BindingFlags.Public | BindingFlags.Static); + object? dataEntities = dataEntitiesProp?.GetValue(null); + if (dataEntities == null) + { + logger.LogInformation("LookupKeys pre-population skipped — dataEntities not loaded yet"); + return false; + } + + PropertyInfo? entityArrayProp = dataEntities.GetType().GetProperty("entity"); + entityArray = entityArrayProp?.GetValue(dataEntities) as System.Collections.IEnumerable; + if (entityArray == null) + { + logger.LogDebug("LookupKeys pre-population skipped — entity array not found"); + return false; + } + + return true; + } + + /// + /// Walks every entity/record in the package and applies both + /// pre-population strategies. + /// + private static Result PrepopulateRecords( + System.Collections.IEnumerable entityArray, + object lookupKeys, + MethodInfo tryAdd) + { + int count = 0; + int entityCount = 0; + + foreach (object? entity in entityArray) + { + if (entity == null) continue; + entityCount++; + + string? entityName = entity.GetType().GetProperty("name")?.GetValue(entity) as string; + if (string.IsNullOrEmpty(entityName)) continue; + + if (entity.GetType().GetProperty("records")?.GetValue(entity) + is not System.Collections.IEnumerable records) + { + continue; + } + + count += PrepopulateEntityRecords(entityName, records, lookupKeys, tryAdd); + } + + return new Result(count, entityCount); + } + + private static int PrepopulateEntityRecords( + string entityName, + System.Collections.IEnumerable records, + object lookupKeys, + MethodInfo tryAdd) + { + int count = 0; + PropertyInfo? idProp = null; + PropertyInfo? newIdProp = null; + + foreach (object? record in records) + { + if (record == null) continue; + + // Cache property lookups after first record. + if (idProp == null) + { + idProp = record.GetType().GetProperty("id"); + newIdProp = record.GetType().GetProperty("newId"); + } + + string? id = idProp?.GetValue(record) as string; + if (string.IsNullOrEmpty(id)) continue; + if (!Guid.TryParse(id, out Guid guid)) continue; + + // Strategy A: pre-seed LookupKeys so FindEntity() short-circuits + // at the cache check (line 2851 of ImportCrmEntityActions.cs). + string key = string.Concat(entityName, ":", id); + tryAdd.Invoke(lookupKeys, [key, guid]); + + // Strategy B: set newId = id on the record object so that even + // if the LookupKeys check misses, FindEntity() at line 2912 finds + // a non-empty newId and returns without a server round-trip. + // CMT preserves source GUIDs (UpsertRequest.Target.Id = source GUID), + // so newId == id is always correct for package-internal references. + if (newIdProp != null) + { + string? existingNewId = newIdProp.GetValue(record) as string; + if (string.IsNullOrWhiteSpace(existingNewId)) + newIdProp.SetValue(record, id); + } + + count++; + } + + return count; + } +}