From 4c40ec150c63e6d61b45b374f40080f3dc9d26d6 Mon Sep 17 00:00:00 2001 From: Weng Xuetian Date: Mon, 13 Jul 2026 18:55:57 -0700 Subject: [PATCH] Programatically generate partial final fuzzy entries / inner segement. This would introduce quite some new entries: e.g. we -> wei, wen, weng The rule to partial final entries: it's not valid, if final of partial final is longer than 2, its (length-1) prefix also need to be invalid pinyin: e.g. sua -> su is valid, so not added to partial final. e.g. ton -> to is invalid, so ton added to partial final. With that we may add more inner fuzzy, so simply generate inner fuzzy programatically. Fix #136 --- src/libime/pinyin/pinyindata.cpp | 157 +++++++++++++++++++++++----- src/libime/pinyin/pinyinencoder.cpp | 8 ++ test/testpinyinencoder.cpp | 12 +++ 3 files changed, 153 insertions(+), 24 deletions(-) diff --git a/src/libime/pinyin/pinyindata.cpp b/src/libime/pinyin/pinyindata.cpp index 2172bf9..bba70bb 100644 --- a/src/libime/pinyin/pinyindata.cpp +++ b/src/libime/pinyin/pinyindata.cpp @@ -6,9 +6,13 @@ #include "pinyindata.h" #include +#include +#include +#include #include #include #include +#include #include #include #include @@ -21,6 +25,95 @@ namespace libime { +namespace { + +void applyPartialFinalFuzzy(PinyinMap &map) { + // There are some finals won't generate any entries, just skip them from + // map. + const auto partialFinalFuzzyMap = + std::to_array>>({ + {"a", + {PinyinFinal::AN, PinyinFinal::AI, PinyinFinal::AO, + PinyinFinal::ANG}}, + {"e", + {PinyinFinal::EI, PinyinFinal::EN, PinyinFinal::ENG, + PinyinFinal::ER}}, + {"i", + {PinyinFinal::IA, PinyinFinal::IAN, PinyinFinal::IAO, + PinyinFinal::IANG, PinyinFinal::IE, PinyinFinal::IN, + PinyinFinal::ING, PinyinFinal::IONG, PinyinFinal::IU}}, + {"ia", {PinyinFinal::IAN, PinyinFinal::IAO, PinyinFinal::IANG}}, + {"ion", {PinyinFinal::IONG}}, + {"o", {PinyinFinal::ONG, PinyinFinal::OU}}, + {"on", {PinyinFinal::ONG}}, + +#if 0 + // Item won't generate anything below. + {"an", {PinyinFinal::ANG}}, + {"en", {PinyinFinal::ENG}}, + {"ian", {PinyinFinal::IANG}}, + {"in", {PinyinFinal::ING}}, + {"io", {PinyinFinal::IONG}}, + + {"u", + {PinyinFinal::UA, PinyinFinal::UAI, PinyinFinal::UAN, + PinyinFinal::UANG, PinyinFinal::UE, PinyinFinal::UI, + PinyinFinal::UN, PinyinFinal::UO}}, + {"ua", {PinyinFinal::UAI, PinyinFinal::UAN, PinyinFinal::UANG}}, + {"uan", {PinyinFinal::UANG}}, + {"v", {PinyinFinal::VE}}, +#endif + }); + std::vector newEntries; + std::map statistic; + for (const auto initial : std::views::iota(PinyinEncoder::firstInitial, + PinyinEncoder::lastInitial) | + std::views::transform([](auto i) { + return static_cast(i); + })) { + if (initial == PinyinInitial::Zero) { + continue; + } + for (const auto &[str, finals] : partialFinalFuzzyMap) { + assert(std::ranges::all_of(finals, [&str](auto f) { + const auto &finalString = PinyinEncoder::finalToString(f); + return finalString.starts_with(str) && + finalString.size() > str.size(); + })); + const auto &initialString = PinyinEncoder::initialToString(initial); + const auto partialFinalString = initialString + str; + bool ignore = false; + for (size_t i = std::max(initialString.size() + 1, + partialFinalString.size() - 1); + i <= partialFinalString.size(); i++) { + if (map.contains(partialFinalString.substr(0, i))) { + ignore = true; + break; + } + } + if (ignore) { + continue; + } + for (auto final : finals) { + auto pinyin = + PinyinEncoder::initialFinalToPinyinString(initial, final); + if (auto iter = map.find(pinyin); + iter != map.end() && + iter->flags() == PinyinFuzzyFlag::None) { + newEntries.push_back( + PinyinEntry(partialFinalString.data(), initial, final, + PinyinFuzzyFlag::PartialFinal)); + } + } + } + } + for (const auto &newEntry : newEntries) { + FCITX_ASSERT(map.insert(newEntry).second); + } +} + +} // namespace + const std::vector &getEncodedInitialFinal() { static const auto encodedInitialFinal = []() { std::vector a; @@ -164,32 +257,46 @@ getInnerSegment() { const InnerSegmentMap &getInnerSegmentV2() { static const InnerSegmentMap innerSegment = []() { - InnerSegmentMap innerSegmentV2; - for (const auto &[key, value] : getInnerSegment()) { - innerSegmentV2[key].push_back(value); - } + InnerSegmentMap innerSegmentV2Generate; + const auto &pinyinMap = getPinyinMapV2(); + for (const auto &entry : getPinyinMapV2()) { + if (entry.pinyin().size() < 3) { + continue; + } + if (innerSegmentV2Generate.contains(entry.pinyin())) { + continue; + } + // We want to support 2 + 1, 2 + 2, 3 + 1, 2 + 3, 3 + 2, 3 + 3. + for (size_t i = 2; i <= 3 && i < entry.pinyin().size(); i++) { + if (entry.pinyin().size() <= i) { + continue; + } + auto part1 = entry.pinyinView().substr(0, i); + auto part2 = entry.pinyinView().substr(i); + if (part2 == "ng") { + continue; + } + auto range1 = pinyinMap.equal_range(part1); + if (!std::any_of( + range1.first, range1.second, [&](const auto &entry) { + return entry.flags() == PinyinFuzzyFlag::None; + })) { + continue; + } + auto range2 = pinyinMap.equal_range(part2); + if (!std::any_of( + range2.first, range2.second, [&](const auto &entry) { + return entry.flags() == PinyinFuzzyFlag::None; + })) { + continue; + } - for (const auto &newItem : std::vector< - std::pair>>{ - {"qiao", {"qia", "o"}}, - {"niao", {"nia", "o"}}, - {"liao", {"lia", "o"}}, - {"zhuo", {"zhu", "o"}}, - {"diao", {"dia", "o"}}, - {"shao", {"sha", "o"}}, - {"xiao", {"xia", "o"}}, - {"zhua", {"zhu", "a"}}, - {"shuo", {"shu", "o"}}, - {"shua", {"shu", "a"}}, - {"zhao", {"zha", "o"}}, - {"jiao", {"jia", "o"}}, - {"chuo", {"chu", "o"}}, - {"chua", {"chu", "a"}}, - {"chao", {"cha", "o"}}, - }) { - innerSegmentV2[newItem.first].push_back(newItem.second); + innerSegmentV2Generate[entry.pinyin()].push_back( + {std::string(part1), std::string(part2)}); + } } - return innerSegmentV2; + + return innerSegmentV2Generate; }(); return innerSegment; @@ -1417,6 +1524,8 @@ const PinyinMap &getPinyinMapV2() { } } + applyPartialFinalFuzzy(filtered); + for (auto fz : {PinyinFuzzyFlag::U_OU, PinyinFuzzyFlag::IN_ING, PinyinFuzzyFlag::EN_ENG, PinyinFuzzyFlag::AN_ANG, PinyinFuzzyFlag::UAN_UANG, PinyinFuzzyFlag::IAN_IANG, diff --git a/src/libime/pinyin/pinyinencoder.cpp b/src/libime/pinyin/pinyinencoder.cpp index 5305d7d..1aea1bd 100644 --- a/src/libime/pinyin/pinyinencoder.cpp +++ b/src/libime/pinyin/pinyinencoder.cpp @@ -322,6 +322,14 @@ PinyinEncoder::parseUserPinyin(std::string userPinyin, auto iter = innerSegments.find(nextPinyin); if (iter != innerSegments.end()) { for (const auto &innerSeg : iter->second) { + if (innerSeg.second == "n") { + bool accept = + (top + nextSize[i] < pinyin.size()) || + (i == 0 && nNextSize == 1); + if (!accept) { + continue; + } + } result.addNext(top, top + innerSeg.first.size()); result.addNext(top + innerSeg.first.size(), diff --git a/test/testpinyinencoder.cpp b/test/testpinyinencoder.cpp index b7cd152..cda1b52 100644 --- a/test/testpinyinencoder.cpp +++ b/test/testpinyinencoder.cpp @@ -279,6 +279,18 @@ int main() { graph = PinyinEncoder::parseUserPinyin("suang", &profile, PinyinFuzzyFlag::Correction); dfs(graph, {"suan", "g"}); + + graph = PinyinEncoder::parseUserPinyin( + "to", &profile, + {PinyinFuzzyFlag::PartialFinal, PinyinFuzzyFlag::InnerShort}); + dfs(graph, {"to"}); + graph = PinyinEncoder::parseUserPinyin( + "ton", &profile, + {PinyinFuzzyFlag::PartialFinal, PinyinFuzzyFlag::InnerShort}); + dfs(graph, {"ton"}); + graph = PinyinEncoder::parseUserPinyin("to", &profile, + PinyinFuzzyFlag::InnerShort); + dfs(graph, {"t", "o"}); } {