From 738e07bfdb8b3fed51c0214416ecbc91d6b52c33 Mon Sep 17 00:00:00 2001 From: Taichi Sato Date: Tue, 22 Sep 2026 22:39:32 +0900 Subject: [PATCH] Deduplicate sub-compound keywords and tighten auto-extraction limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Suppress sub-compound splitting of 4-character kanji compounds - Trim suffix stopwords and filter '~時' temporal suffixes - Expand JP_STOP_WORDS with common procedural and state terms - Deduplicate substrings (longest-match priority) in auto-extracted keywords - Reduce MAX_AUTO_KEYWORDS from 15 to 8 and update regen default threshold --- src/keywords.rs | 201 ++++++++++++++++++++++++++++++++++++++++++------ src/main.rs | 2 +- 2 files changed, 180 insertions(+), 23 deletions(-) diff --git a/src/keywords.rs b/src/keywords.rs index 154c6ae..659b664 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -18,13 +18,18 @@ const STOP_WORDS: &[&str] = &[ /// covers the entire title/content, so keywords only need to be the terms that best /// represent the entry — an uncapped dump of every word just adds noise to keyword /// search and duplicate detection. -pub const MAX_AUTO_KEYWORDS: usize = 15; +pub const MAX_AUTO_KEYWORDS: usize = 8; /// Japanese general stopwords for candidate filtering. const JP_STOP_WORDS: &[&str] = &[ "場合", "設定", "追加", "確認", "対応", "実行", "作成", "更新", "取得", "処理", "使用", "問題", "修正", "機能", "変更", "必要", "可能", "利用", "対象", "詳細", "理由", "手順", "記述", "指定", "管理", "表示", "発生", "関係", "存在", "関連", "完了", "状況", "現在", + // General procedural / relational stopwords + "自動", "手動", "検討", "概要", "目的", "方法", "方針", "結果", "注意", "内容", "前提", "実施", + "導入", "参照", "共有", "反映", "保存", "調査", "動作", "連携", "提供", "定義", "考慮", "適用", + "部分", "全体", "自身", "以降", "以前", "程度", "通常", "以下", "以上", "未満", "各種", "複数", + "一覧", ]; /// A term occurring in the title counts this many times a content occurrence. @@ -45,19 +50,24 @@ pub fn extract_keywords_auto( if !candidates.is_empty() && let Ok(ranked) = client.filter_keywords(title, content, &candidates) { - // Keep terms with score >= 0.50, capped at MAX_AUTO_KEYWORDS - let mut selected: Vec = ranked + // Filter terms with score >= 0.50 + let qualified: Vec = ranked .iter() .filter(|(_, score)| *score >= 0.50) .map(|(kw, _)| kw.clone()) - .take(MAX_AUTO_KEYWORDS) .collect(); - // If threshold filtered everything out, take top 5 from Laya's ranked results + let mut selected = deduplicate_substrings(&qualified); + selected.truncate(MAX_AUTO_KEYWORDS); + + // If threshold filtered everything out, take top from Laya's ranked results if selected.is_empty() && !ranked.is_empty() { - selected = ranked.into_iter().take(5).map(|(kw, _)| kw).collect(); + let fallback: Vec = ranked.into_iter().map(|(kw, _)| kw).collect(); + selected = deduplicate_substrings(&fallback); + selected.truncate(MAX_AUTO_KEYWORDS.min(5)); } else if selected.is_empty() && !candidates.is_empty() { - selected = candidates.into_iter().take(5).collect(); + selected = deduplicate_substrings(&candidates); + selected.truncate(MAX_AUTO_KEYWORDS.min(5)); } if !selected.is_empty() { @@ -70,6 +80,65 @@ pub fn extract_keywords_auto( extract_keywords(title, content) } +/// Check if `shorter` is a sub-compound / substring of `longer`. +/// +/// For Japanese terms (kanji, katakana), substring containment indicates +/// a sub-compound relationship (e.g. "排他制御" contains "排他" / "制御", +/// "アクセストークン" contains "トークン"). +/// +/// For pure ASCII words, simple containment would produce false positives +/// (e.g. "format" in "information"). We only match if `longer` contains `shorter` +/// as a delimited token (separated by '_' or '-') or as a prefix/suffix +/// with a substantive remainder (>= 3 chars). +pub fn is_subword(longer: &str, shorter: &str) -> bool { + if longer.len() <= shorter.len() || longer == shorter { + return false; + } + + let has_non_ascii = !longer.is_ascii() || !shorter.is_ascii(); + if has_non_ascii { + return longer.contains(shorter); + } + + // Pure ASCII + for part in longer.split(['_', '-']) { + if part.eq_ignore_ascii_case(shorter) { + return true; + } + } + + let l_lower = longer.to_ascii_lowercase(); + let s_lower = shorter.to_ascii_lowercase(); + if l_lower.starts_with(&s_lower) { + let remainder = &l_lower[s_lower.len()..]; + if remainder.len() >= 3 { + return true; + } + } + if l_lower.ends_with(&s_lower) { + let remainder = &l_lower[..l_lower.len() - s_lower.len()]; + if remainder.len() >= 3 { + return true; + } + } + + false +} + +/// Remove shorter keywords that are sub-compounds of a longer keyword present in the list. +/// Preserves the original relative order of non-subword keywords. +pub fn deduplicate_substrings(keywords: &[String]) -> Vec { + keywords + .iter() + .filter(|kw| { + !keywords + .iter() + .any(|other| other.as_str() != kw.as_str() && is_subword(other, kw)) + }) + .cloned() + .collect() +} + /// Extract candidate keywords for Laya ranking, including ASCII words, katakana, /// and Japanese kanji compounds (2-6 chars). pub fn extract_candidates_for_laya(title: &str, content: &str) -> Vec { @@ -117,27 +186,34 @@ fn score_text_with_kanji(text: &str, weight: u32, scores: &mut HashMap = s.chars().collect(); let len = chars.len(); + + // 1. If compound ends with '時' (e.g. 発生時 -> skip because 発生 is a stop word; 失効時 -> 失効) + if len >= 3 && chars.last() == Some(&'時') { + let stem: String = chars[..len - 1].iter().collect(); + if !JP_STOP_WORDS.contains(&stem.as_str()) && stem.chars().count() >= 2 { + *scores.entry(stem).or_insert(0) += weight; + } + continue; + } + + // 2. If compound (>= 4 chars) ends with a 2-char stop word (e.g. 排他制御手順 -> 排他制御, 自動更新処理 -> 自動更新) if len >= 4 { - // Sub-compound candidates: 2-char and 4-char prefixes/suffixes - for w in [2, 4] { - if w < len { - let prefix: String = chars[..w].iter().collect(); - if !JP_STOP_WORDS.contains(&prefix.as_str()) { - *scores.entry(prefix).or_insert(0) += weight; - } - let suffix: String = chars[len - w..].iter().collect(); - if !JP_STOP_WORDS.contains(&suffix.as_str()) { - *scores.entry(suffix).or_insert(0) += weight; - } + let suffix: String = chars[len - 2..].iter().collect(); + if JP_STOP_WORDS.contains(&suffix.as_str()) { + let stem: String = chars[..len - 2].iter().collect(); + if !JP_STOP_WORDS.contains(&stem.as_str()) && stem.chars().count() >= 2 { + *scores.entry(stem).or_insert(0) += weight; } + continue; } } + + // 3. Normal compound if not in JP_STOP_WORDS + if !JP_STOP_WORDS.contains(&s) { + *scores.entry(s.to_string()).or_insert(0) += weight; + } } } @@ -333,4 +409,85 @@ mod tests { assert!(!kws.is_empty()); assert!(kws.contains(&"session".to_string())); } + + #[test] + fn test_is_subword() { + // Japanese compounds + assert!(is_subword("アクセストークン", "トークン")); + assert!(is_subword("排他制御", "排他")); + assert!(is_subword("排他制御", "制御")); + assert!(is_subword("自動更新", "自動")); + assert!(!is_subword("トークン", "アクセストークン")); + assert!(!is_subword("排他制御", "排他制御")); + + // ASCII words + assert!(is_subword("sessionmanager", "session")); + assert!(is_subword("sessionmanager", "manager")); + assert!(is_subword("access_token", "token")); + assert!(!is_subword("information", "format")); + assert!(!is_subword("token", "token")); + } + + #[test] + fn test_deduplicate_substrings() { + let input = vec![ + "oauth2".to_string(), + "アクセストークン".to_string(), + "トークン".to_string(), + "バックグラウンド".to_string(), + "リトライ".to_string(), + "再取得".to_string(), + "制御".to_string(), + "排他".to_string(), + "排他制御".to_string(), + "更新処理".to_string(), + "自動".to_string(), + "自動更新".to_string(), + "通信".to_string(), + ]; + let deduped = deduplicate_substrings(&input); + assert!(deduped.contains(&"アクセストークン".to_string())); + assert!(!deduped.contains(&"トークン".to_string())); + assert!(deduped.contains(&"排他制御".to_string())); + assert!(!deduped.contains(&"排他".to_string())); + assert!(!deduped.contains(&"制御".to_string())); + assert!(deduped.contains(&"自動更新".to_string())); + assert!(!deduped.contains(&"自動".to_string())); + assert!(deduped.contains(&"oauth2".to_string())); + assert!(deduped.contains(&"通信".to_string())); + } + + #[test] + fn test_user_example_candidate_extraction() { + let title = "OAuth2トークンの自動更新処理"; + let content = "401エラー発生時にバックグラウンドで新しいアクセストークンを再取得して通信をリトライする排他制御フロー。"; + let candidates = extract_candidates_for_laya(title, content); + + // Sub-compound suppression: 4-char compounds must not be split into 2-char pieces + assert!(candidates.contains(&"排他制御".to_string())); + assert!(!candidates.contains(&"排他".to_string())); + assert!(!candidates.contains(&"制御".to_string())); + + // Suffix stopword trimming: "自動更新処理" -> "自動更新", and "自動" stopword + assert!(candidates.contains(&"自動更新".to_string())); + assert!(!candidates.contains(&"自動更新処理".to_string())); + assert!(!candidates.contains(&"自動".to_string())); + + // '〜時' suffix attached to stopword "発生" -> completely skipped + assert!(!candidates.contains(&"発生時".to_string())); + assert!(!candidates.contains(&"発生".to_string())); + + // Katakana and ASCII + assert!(candidates.contains(&"oauth2".to_string())); + assert!(candidates.contains(&"アクセストークン".to_string())); + assert!(candidates.contains(&"バックグラウンド".to_string())); + assert!(candidates.contains(&"リトライ".to_string())); + assert!(candidates.contains(&"再取得".to_string())); + assert!(candidates.contains(&"通信".to_string())); + + // After deduplication, "トークン" is excluded because "アクセストークン" exists + let deduped = deduplicate_substrings(&candidates); + assert!(deduped.contains(&"アクセストークン".to_string())); + assert!(!deduped.contains(&"トークン".to_string())); + } } diff --git a/src/main.rs b/src/main.rs index 3a42423..1deb360 100644 --- a/src/main.rs +++ b/src/main.rs @@ -258,7 +258,7 @@ enum Commands { #[arg(long, requires = "regen")] all: bool, /// With --regen: entries with more keywords than this are considered noisy - #[arg(long, default_value = "15", requires = "regen")] + #[arg(long, default_value = "8", requires = "regen")] threshold: usize, /// With --regen: show what would change without writing #[arg(long, requires = "regen")]