Handle morph-type tokens: filtering / persistence

backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/SortingTests.cs

@@ -1,4 +1,7 @@
+using FwDataMiniLcmBridge.Api;
+using FwDataMiniLcmBridge.LcmUtils;
 using FwDataMiniLcmBridge.Tests.Fixtures;
+using MiniLcm.Models;
 
 namespace FwDataMiniLcmBridge.Tests.MiniLcmTests;
 
@@ -9,4 +12,42 @@ protected override Task<IMiniLcmApi> NewApi()
     {
         return Task.FromResult<IMiniLcmApi>(fixture.NewProjectApi("sorting-test", "en", "en"));
     }
+
+    [Theory]
+    [InlineData("aaaa", SortField.Headword)] // FTS
+    [InlineData("a", SortField.Headword)] // non-FTS
+    [InlineData("aaaa", SortField.SearchRelevance)] // FTS
+    [InlineData("a", SortField.SearchRelevance)] // non-FTS
+    public async Task SecondaryOrder_DefaultsToStem(string query, SortField sortField)
+    {
+        var unknownMorphTypeEntryId = Guid.NewGuid();
+        Entry[] expected = [
+            new() { Id = unknownMorphTypeEntryId, LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.Unknown }, // SecondaryOrder defaults to Stem = 1
+            new() { Id = Guid.NewGuid(), LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.BoundStem }, // SecondaryOrder = 2
+            new() { Id = Guid.NewGuid(), LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.Suffix }, // SecondaryOrder = 6
+        ];
+
+        var ids = expected.Select(e => e.Id).ToHashSet();
+
+        foreach (var entry in Faker.Faker.Random.Shuffle(expected))
+            await Api.CreateEntry(entry);
+
+        var fwDataApi = (BaseApi as FwDataMiniLcmApi)!;
+        await fwDataApi.Cache.DoUsingNewOrCurrentUOW("Clear morph type",
+            "Revert morph type",
+            () =>
+            {
+                // the fwdata api doesn't allow creating entries with MorphType.Other or Unknown, so we force it
+                var unknownMorphTypeEntry = fwDataApi.EntriesRepository.GetObject(unknownMorphTypeEntryId);
+                unknownMorphTypeEntry.LexemeFormOA.MorphTypeRA = null;
+                return ValueTask.CompletedTask;
+            });
+
+        var results = (await Api.SearchEntries(query, new(new(sortField))).ToArrayAsync())
+            .Where(e => ids.Contains(e.Id))
+            .ToList();
+
+        results.Should().BeEquivalentTo(expected,
+            options => options.WithStrictOrdering());
+    }
 }

backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs

@@ -652,7 +652,7 @@ private Entry FromLexEntry(ILexEntry entry)
     {
         try
         {
-            return new Entry
+            var result = new Entry
             {
                 Id = entry.Guid,
                 Note = FromLcmMultiString(entry.Comment),
@@ -670,6 +670,7 @@ private Entry FromLexEntry(ILexEntry entry)
                 // ILexEntry.PublishIn is a virtual property that inverts DoNotPublishInRC against all publications
                 PublishIn = entry.PublishIn.Select(FromLcmPossibility).ToList(),
             };
+            return result;
         }
         catch (Exception e)
         {
@@ -725,24 +726,22 @@ private ComplexFormComponent ToEntryReference(ILexEntry component, ILexEntry com
         return new ComplexFormComponent
         {
             ComponentEntryId = component.Guid,
-            ComponentHeadword = component.LexEntryHeadwordOrUnknown(),
+            ComponentHeadword = component.LexEntryHeadwordOrUnknown(applyMorphTokens: false), // match CRDT for now
             ComplexFormEntryId = complexEntry.Guid,
-            ComplexFormHeadword = complexEntry.LexEntryHeadwordOrUnknown(),
+            ComplexFormHeadword = complexEntry.LexEntryHeadwordOrUnknown(applyMorphTokens: false), // match CRDT for now
             Order = Order(component, complexEntry)
         };
     }
 
-
-
     private ComplexFormComponent ToSenseReference(ILexSense componentSense, ILexEntry complexEntry)
     {
         return new ComplexFormComponent
         {
             ComponentEntryId = componentSense.Entry.Guid,
             ComponentSenseId = componentSense.Guid,
-            ComponentHeadword = componentSense.Entry.LexEntryHeadwordOrUnknown(),
+            ComponentHeadword = componentSense.Entry.LexEntryHeadwordOrUnknown(applyMorphTokens: false), // match CRDT for now
             ComplexFormEntryId = complexEntry.Guid,
-            ComplexFormHeadword = complexEntry.LexEntryHeadwordOrUnknown(),
+            ComplexFormHeadword = complexEntry.LexEntryHeadwordOrUnknown(applyMorphTokens: false), // match CRDT for now
             Order = Order(componentSense, complexEntry)
         };
     }
@@ -939,12 +938,13 @@ private IEnumerable<ILexEntry> GetFilteredAndSortedEntries(Func<ILexEntry, bool>
     private IEnumerable<ILexEntry> ApplySorting(SortOptions order, IEnumerable<ILexEntry> entries, string? query)
     {
         var sortWs = GetWritingSystemHandle(order.WritingSystem, WritingSystemType.Vernacular);
+        var stemSecondaryOrder = MorphTypeRepository.GetObject(MoMorphTypeTags.kguidMorphStem).SecondaryOrder;
         if (order.Field == SortField.SearchRelevance)
         {
-            return entries.ApplyRoughBestMatchOrder(order, sortWs, query);
+            return entries.ApplyRoughBestMatchOrder(order, sortWs, stemSecondaryOrder, query);
         }
 
-        return order.ApplyOrder(entries, e => e.LexEntryHeadword(sortWs));
+        return entries.ApplyHeadwordOrder(order, sortWs, stemSecondaryOrder);
     }
 
     public IAsyncEnumerable<Entry> SearchEntries(string query, QueryOptions? options = null)
@@ -956,7 +956,7 @@ public IAsyncEnumerable<Entry> SearchEntries(string query, QueryOptions? options
     private Func<ILexEntry, bool>? EntrySearchPredicate(string? query = null)
     {
         if (string.IsNullOrEmpty(query)) return null;
-        return entry => entry.CitationForm.SearchValue(query) ||
+        return entry => entry.SearchHeadWord(query) || // CitationForm.SearchValue would be redundant
                         entry.LexemeFormOA?.Form.SearchValue(query) is true ||
                         entry.AllSenses.Any(s => s.Gloss.SearchValue(query));
     }

backend/FwLite/FwDataMiniLcmBridge/Api/LcmHelpers.cs

@@ -11,7 +11,7 @@ namespace FwDataMiniLcmBridge.Api;
 
 internal static class LcmHelpers
 {
-    internal static string? LexEntryHeadword(this ILexEntry entry, int? ws = null)
+    internal static string? LexEntryHeadword(this ILexEntry entry, int? ws = null, bool applyMorphTokens = true)
     {
         var citationFormTs =
             ws.HasValue ? entry.CitationForm.get_String(ws.Value)
@@ -27,15 +27,33 @@ internal static class LcmHelpers
             : null;
         var lexemeForm = lexemeFormTs?.Text?.Trim(WhitespaceChars);
 
-        return lexemeForm;
+        if (string.IsNullOrEmpty(lexemeForm) || !applyMorphTokens) return lexemeForm;
+
+        var morphType = entry.LexemeFormOA?.MorphTypeRA;
+        var leading = morphType?.Prefix ?? "";
+        var trailing = morphType?.Postfix ?? "";
+        return (leading + lexemeForm + trailing).Trim(WhitespaceChars);
     }
 
-    internal static string LexEntryHeadwordOrUnknown(this ILexEntry entry, int? ws = null)
+    internal static string LexEntryHeadwordOrUnknown(this ILexEntry entry, int? ws = null, bool applyMorphTokens = true)
     {
-        var headword = entry.LexEntryHeadword(ws);
+        var headword = entry.LexEntryHeadword(ws, applyMorphTokens);
         return string.IsNullOrEmpty(headword) ? Entry.UnknownHeadword : headword;
     }
 
+    internal static bool SearchHeadWord(this ILexEntry entry, string value)
+    {
+        foreach (var ws in entry.Cache.ServiceLocator.WritingSystems.VernacularWritingSystems)
+        {
+            var headword = entry.HeadWordForWs(ws.Handle);
+            if (headword is null) continue;
+            var text = headword.Text;
+            if (string.IsNullOrEmpty(text)) continue;
+            if (text.ContainsDiacriticMatch(value)) return true;
+        }
+        return false;
+    }
+
     internal static bool SearchValue(this ITsMultiString multiString, string value)
     {
         for (var i = 0; i < multiString.StringCount; i++)

backend/FwLite/FwDataMiniLcmBridge/Api/Sorting.cs

@@ -6,31 +6,55 @@ namespace FwDataMiniLcmBridge.Api;
 
 internal static class Sorting
 {
+    public static IEnumerable<ILexEntry> ApplyHeadwordOrder(this IEnumerable<ILexEntry> entries, SortOptions order, int sortWsHandle, int stemSecondaryOrder)
+    {
+        if (order.Ascending)
+        {
+            return entries
+                .OrderBy(e => e.LexEntryHeadword(sortWsHandle, applyMorphTokens: false))
+                .ThenBy(e => e.PrimaryMorphType?.SecondaryOrder ?? stemSecondaryOrder)
+                // .ThenBy(e => e.HomographNumber)
+                .ThenBy(e => e.Id.Guid);
+        }
+        else
+        {
+            return entries
+                .OrderByDescending(e => e.LexEntryHeadword(sortWsHandle, applyMorphTokens: false))
+                .ThenByDescending(e => e.PrimaryMorphType?.SecondaryOrder ?? stemSecondaryOrder)
+                // .ThenByDescending(e => e.HomographNumber)
+                .ThenByDescending(e => e.Id.Guid);
+        }
+    }
+
     /// <summary>
     /// Rough emulation of FTS search relevance. Headword matches come first, preferring
     /// prefix matches (e.g. when searching "tan" then "tanan" is before "matan"), then shorter, then alphabetical.
     /// See also: EntrySearchService.FilterAndRank for the FTS-based equivalent in LcmCrdt.
     /// </summary>
-    public static IEnumerable<ILexEntry> ApplyRoughBestMatchOrder(this IEnumerable<ILexEntry> entries, SortOptions order, int sortWsHandle, string? query = null)
+    public static IEnumerable<ILexEntry> ApplyRoughBestMatchOrder(this IEnumerable<ILexEntry> entries, SortOptions order, int sortWsHandle, int stemSecondaryOrder, string? query = null)
     {
-        var projected = entries.Select(e => (Entry: e, Headword: e.LexEntryHeadword(sortWsHandle)));
+        var projected = entries.Select(e => (Entry: e, Headword: e.LexEntryHeadword(sortWsHandle, applyMorphTokens: false)));
         if (order.Ascending)
         {
             return projected
-                .OrderByDescending(x => !string.IsNullOrEmpty(query) && (x.Headword?.ContainsDiacriticMatch(query!) ?? false))
-                .ThenByDescending(x => !string.IsNullOrEmpty(query) && (x.Headword?.StartsWithDiacriticMatch(query!) ?? false))
+                .OrderByDescending(x => !string.IsNullOrEmpty(query) && (x.Headword?.StartsWithDiacriticMatch(query!) ?? false))
+                .ThenByDescending(x => !string.IsNullOrEmpty(query) && (x.Headword?.ContainsDiacriticMatch(query!) ?? false))
                 .ThenBy(x => x.Headword?.Length ?? 0)
                 .ThenBy(x => x.Headword)
+                .ThenBy(x => x.Entry.PrimaryMorphType?.SecondaryOrder ?? stemSecondaryOrder)
+                // .ThenBy(x => x.Entry.HomographNumber)
                 .ThenBy(x => x.Entry.Id.Guid)
                 .Select(x => x.Entry);
         }
         else
         {
             return projected
-                .OrderBy(x => !string.IsNullOrEmpty(query) && (x.Headword?.ContainsDiacriticMatch(query!) ?? false))
-                .ThenBy(x => !string.IsNullOrEmpty(query) && (x.Headword?.StartsWithDiacriticMatch(query!) ?? false))
+                .OrderBy(x => !string.IsNullOrEmpty(query) && (x.Headword?.StartsWithDiacriticMatch(query!) ?? false))
+                .ThenBy(x => !string.IsNullOrEmpty(query) && (x.Headword?.ContainsDiacriticMatch(query!) ?? false))
                 .ThenByDescending(x => x.Headword?.Length ?? 0)
                 .ThenByDescending(x => x.Headword)
+                .ThenByDescending(x => x.Entry.PrimaryMorphType?.SecondaryOrder ?? stemSecondaryOrder)
+                // .ThenByDescending(x => x.Entry.HomographNumber)
                 .ThenByDescending(x => x.Entry.Id.Guid)
                 .Select(x => x.Entry);
         }

backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateMorphTypeProxy.cs

@@ -1,3 +1,4 @@
+using System.Diagnostics.CodeAnalysis;
 using MiniLcm.Models;
 using SIL.LCModel;
 
@@ -8,11 +9,13 @@ public class UpdateMorphTypeProxy : MorphType
     private readonly IMoMorphType _lcmMorphType;
     private readonly FwDataMiniLcmApi _lexboxLcmApi;
 
+    [SetsRequiredMembers]
     public UpdateMorphTypeProxy(IMoMorphType lcmMorphType, FwDataMiniLcmApi lexboxLcmApi)
     {
         _lcmMorphType = lcmMorphType;
         Id = lcmMorphType.Guid;
         _lexboxLcmApi = lexboxLcmApi;
+        Kind = LcmHelpers.FromLcmMorphType(lcmMorphType);
     }
 
     public override MultiString Name