From 820baa3ef219a2a60964b62026606ca3eab508ef Mon Sep 17 00:00:00 2001 From: Yohei <20108042+owayo@users.noreply.github.com> Date: Tue, 13 Oct 2026 07:20:48 +0900 Subject: [PATCH] =?UTF-8?q?test:=20=E8=BE=9E=E6=9B=B8=E7=9B=A3=E6=9F=BB?= =?UTF-8?q?=E3=81=AE=E8=AA=AD=E3=81=BF=E3=81=A8=E6=95=B0=E9=87=8F=E3=83=88?= =?UTF-8?q?=E3=83=BC=E3=82=AF=E3=83=B3=E3=81=AE=E5=9B=9E=E5=B8=B0=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pos.rs | 5 ++- tests/integration.rs | 85 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/src/pos.rs b/src/pos.rs index 9d1a997..4693084 100644 --- a/src/pos.rs +++ b/src/pos.rs @@ -1124,6 +1124,7 @@ mod tests { "することができる。", "示されるものだ。", "雨が降りそうだ。", + "注意,重要。", ]; let mut expected: Vec> = Vec::new(); for (name, mut analyzer) in distributed_dicts() { @@ -1147,7 +1148,9 @@ mod tests { assert!(has(0, "(", CoarsePos::OpenBracket)); assert!(has(0, "!", CoarsePos::Period)); assert!(has(0, "！", CoarsePos::Period)); - assert!(has(0, ",", CoarsePos::Comma)); + assert!(has(4, ",", CoarsePos::Comma)); + // 数量の桁区切りは数詞の表層に含まれ、読点にはしない。 + assert!(has(0, "1,000", CoarsePos::Numeral)); // 「こと」は形式名詞、受け身の「れる」と助動詞の語幹「そう」は助動詞 assert!(has(1, "こと", CoarsePos::FormalNoun)); assert!(has(2, "れる", CoarsePos::AuxVerb)); diff --git a/tests/integration.rs b/tests/integration.rs index e6c2674..065b8f6 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -4165,3 +4165,88 @@ fn test_cli_tokenize_accepts_empty_and_newline_terminated_input() { std::fs::remove_file(&tmp).unwrap(); } + +#[test] +#[ignore = "配布辞書を dict/ に用意して実行する"] +fn test_distributed_compound_noun_pronunciation_keeps_long_vowels() { + for name in ["ipadic-neologd", "ipadic-neologd-sudachi"] { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("dict") + .join(format!("{name}.hsd")); + let mut analyzer = Analyzer::load(&path).unwrap(); + for (surface, reading, pronunciation) in [ + ("初登場", "ハツトウジョウ", "ハツトージョー"), + ( + "匿名流動型犯罪グループ", + "トクメイリュウドウガタハンザイグループ", + "トクメイリュードーガタハンザイグループ", + ), + ("透磁率", "トウジリツ", "トージリツ"), + ] { + let text = format!("{surface}の情報を確認する。"); + let tokens = analyzer.try_tokenize(&text).unwrap(); + let word = &tokens[0]; + assert_eq!(&*word.surface, surface, "{name}: {text}"); + assert_eq!(&*word.base_form, surface); + assert_eq!(&*word.reading, reading); + assert_eq!(&*word.pronunciation, pronunciation); + assert!(word.is_known && word.pos.starts_with("名詞,")); + assert_eq!((word.start, word.end), (0, surface.len())); + assert_eq!(&*tokens[1].surface, "の"); + } + for (surface, pronunciation) in [("思う", "オモウ"), ("迷う", "マヨウ")] { + let tokens = analyzer.try_tokenize(surface).unwrap(); + assert_eq!(tokens.len(), 1); + assert_eq!(&*tokens[0].pronunciation, pronunciation); + } + } +} + +#[test] +#[ignore = "配布辞書を dict/ に用意して実行する"] +fn test_distributed_modern_terms_preserve_grammar_and_ascii_runs() { + for name in ["ipadic-neologd", "ipadic-neologd-sudachi"] { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("dict") + .join(format!("{name}.hsd")); + let mut analyzer = Analyzer::load(&path).unwrap(); + for (surface, base, reading, pos) in [ + ("BUDDiiS", "BUDDiiS", "バディーズ", "名詞,固有名詞,一般,*"), + ("ICEx", "ICEx", "アイス", "名詞,固有名詞,一般,*"), + ("Lienel", "Lienel", "リエネル", "名詞,固有名詞,一般,*"), + ("ABEMA", "ABEMA", "アベマ", "名詞,固有名詞,一般,*"), + ("Abema", "ABEMA", "アベマ", "名詞,固有名詞,一般,*"), + ("閉区間", "閉区間", "ヘイクカン", "名詞,一般,*,*"), + ] { + let text = format!("{surface}の情報を確認する。"); + let tokens = analyzer.try_tokenize(&text).unwrap(); + assert_eq!(&*tokens[0].surface, surface, "{name}: {text}"); + assert_eq!(&*tokens[0].base_form, base); + assert_eq!(&*tokens[0].reading, reading); + assert_eq!(&*tokens[0].pronunciation, reading); + assert_eq!(&*tokens[0].pos, pos); + assert!(tokens[0].is_known); + assert_eq!((tokens[0].start, tokens[0].end), (0, surface.len())); + assert_eq!(&*tokens[1].surface, "の"); + } + let tokens = analyzer.try_tokenize("パラメトライズされた関数。").unwrap(); + assert_eq!(&*tokens[0].surface, "パラメトライズ"); + assert_eq!(&*tokens[0].pos, "名詞,サ変接続,*,*"); + assert_eq!(&*tokens[1].surface, "さ"); + assert!(tokens[1].pos.starts_with("動詞,")); + + let tokens = analyzer.try_tokenize("とはいえ条件を確認する。").unwrap(); + assert_eq!(&*tokens[0].surface, "とはいえ"); + assert_eq!(&*tokens[0].pos, "接続詞,*,*,*"); + assert_eq!(&*tokens[0].reading, "トハイエ"); + assert_eq!(&*tokens[0].pronunciation, "トワイエ"); + + for text in ["ABEMAX", "XABEMA", "Abemas", "ICExamples", "BUDDiiSValue"] { + let tokens = analyzer.try_tokenize(text).unwrap(); + assert_eq!(tokens.len(), 1, "{name}: {text}"); + assert_eq!(&*tokens[0].surface, text); + assert!(!tokens[0].is_known, "{name}: {text}"); + assert_eq!((tokens[0].start, tokens[0].end), (0, text.len())); + } + } +}