Merge pull request #209 from send/refactor/absorb-reranker-costs

refactor: absorb 3 reranker heuristics into compile-time dictionary costs

Author: send

engine/crates/lex-cli/src/bin/dictool.rs

@@ -30,6 +30,9 @@ enum Command {
         input_dir: String,
         /// Output file
         output_file: String,
+        /// Mozc id.def for compile-time cost adjustments (person name, pronoun)
+        #[arg(long)]
+        id_def: Option<String>,
     },
     /// Compile connection matrix
     CompileConn {
@@ -221,7 +224,8 @@ fn main() {
             source,
             input_dir,
             output_file,
-        } => dict_ops::compile(&source, &input_dir, &output_file),
+            id_def,
+        } => dict_ops::compile(&source, &input_dir, &output_file, id_def.as_deref()),
         Command::CompileConn {
             input_txt,
             output_file,

engine/crates/lex-cli/src/commands/dict_ops.rs

@@ -1,6 +1,6 @@
 use std::collections::{HashMap, HashSet};
 use std::fs;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::process;
 
 use crate::dict_source;
@@ -29,7 +29,12 @@ pub fn fetch(source_name: &str, output_dir: &str) {
     );
 }
 
-pub fn compile(source_name: &str, input_dir: &str, output_file: &str) {
+/// Cost offsets applied at dictionary compile time to eliminate reranker heuristics.
+const PERSON_NAME_COST_OFFSET: i16 = 2000;
+const PRONOUN_COST_OFFSET: i16 = -3500;
+const NON_INDEPENDENT_KANJI_COST_OFFSET: i16 = 1500;
+
+pub fn compile(source_name: &str, input_dir: &str, output_file: &str, id_def: Option<&str>) {
     let dict_source = dict_source::from_name(source_name).unwrap_or_else(|| {
         eprintln!("Error: unknown source '{source_name}' (available: mozc)");
         process::exit(1);
@@ -42,11 +47,58 @@ pub fn compile(source_name: &str, input_dir: &str, output_file: &str) {
     }
 
     eprintln!("Source: {source_name}");
-    let entries = die!(
+    let mut entries = die!(
         dict_source.parse_dir(input_path),
         "Error parsing dictionary: {}"
     );
 
+    // Apply compile-time cost offsets based on morpheme roles.
+    // Auto-detect id.def in input_dir if --id-def is not specified.
+    let id_def_path = id_def.map(PathBuf::from).or_else(|| {
+        let auto = input_path.join("id.def");
+        if auto.is_file() {
+            eprintln!("Auto-detected id.def at {}", auto.display());
+            Some(auto)
+        } else {
+            None
+        }
+    });
+    if let Some(id_def_path) = &id_def_path {
+        let roles = die!(
+            pos_map::morpheme_roles(id_def_path),
+            "Error loading morpheme roles: {}"
+        );
+        let mut adjusted = 0usize;
+        for entry_list in entries.values_mut() {
+            for entry in entry_list.iter_mut() {
+                let id = entry.left_id as usize;
+                if id >= roles.len() {
+                    eprintln!(
+                        "Warning: left_id {} out of roles table range ({}), skipping entry '{}'",
+                        id,
+                        roles.len(),
+                        entry.surface
+                    );
+                    continue;
+                }
+                let role = roles[id];
+                let offset = match role {
+                    pos_map::ROLE_PERSON_NAME => PERSON_NAME_COST_OFFSET,
+                    pos_map::ROLE_PRONOUN => PRONOUN_COST_OFFSET,
+                    pos_map::ROLE_NON_INDEPENDENT
+                        if entry.surface.chars().any(lex_core::unicode::is_kanji) =>
+                    {
+                        NON_INDEPENDENT_KANJI_COST_OFFSET
+                    }
+                    _ => continue,
+                };
+                entry.cost = entry.cost.saturating_add(offset);
+                adjusted += 1;
+            }
+        }
+        eprintln!("Adjusted {adjusted} entries (person_name: +{PERSON_NAME_COST_OFFSET}, pronoun: {PRONOUN_COST_OFFSET}, non_independent_kanji: +{NON_INDEPENDENT_KANJI_COST_OFFSET})");
+    }
+
     let reading_count = entries.len();
     let entry_count: usize = entries.values().map(|v| v.len()).sum();
 

engine/crates/lex-core/src/converter/explain.rs

@@ -75,10 +75,6 @@ pub struct ExplainSegment {
     pub script_cost: i64,
     /// Connection cost from BOS or previous segment.
     pub connection_cost: i64,
-    /// Non-independent kanji penalty applied.
-    pub non_independent_kanji_penalty: i64,
-    /// Pronoun cost bonus applied (positive value, subtracted from cost).
-    pub pronoun_bonus: i64,
     /// Te-form kanji penalty applied.
     pub te_form_kanji_penalty: i64,
     /// Single-char kanji content-word penalty applied.
@@ -120,15 +116,13 @@ fn explain_segments(
             } else {
                 None
             };
-            let (ni_penalty, p_bonus, te_penalty, sc_penalty) = if let Some(c) = conn {
+            let (te_penalty, sc_penalty) = if let Some(c) = conn {
                 (
-                    reranker::non_independent_kanji_penalty(seg, c),
-                    reranker::pronoun_bonus(seg, c),
                     reranker::te_form_kanji_penalty(prev_seg, seg, c),
                     reranker::single_char_kanji_penalty(seg, i, &scored.segments, c, Some(dict)),
                 )
             } else {
-                (0, 0, 0, 0)
+                (0, 0)
             };
             ExplainSegment {
                 reading: seg.reading.clone(),
@@ -137,8 +131,6 @@ fn explain_segments(
                 segment_penalty: settings().cost.segment_penalty,
                 script_cost: script_cost(&seg.surface, seg.reading.chars().count()),
                 connection_cost: connection,
-                non_independent_kanji_penalty: ni_penalty,
-                pronoun_bonus: p_bonus,
                 te_form_kanji_penalty: te_penalty,
                 single_char_kanji_penalty: sc_penalty,
                 left_id: seg.left_id,
@@ -289,16 +281,6 @@ pub fn format_text(result: &ExplainResult) -> String {
                 seg_label
             };
             let conn_label = if j == 0 { "BOS->" } else { "conn=" };
-            let ni_str = if seg.non_independent_kanji_penalty > 0 {
-                format!(" ni_kanji={:<+6}", seg.non_independent_kanji_penalty)
-            } else {
-                String::new()
-            };
-            let pronoun_str = if seg.pronoun_bonus > 0 {
-                format!(" pronoun={:<+6}", -(seg.pronoun_bonus))
-            } else {
-                String::new()
-            };
             let te_str = if seg.te_form_kanji_penalty > 0 {
                 format!(" teK={:<+6}", seg.te_form_kanji_penalty)
             } else {
@@ -310,16 +292,14 @@ pub fn format_text(result: &ExplainResult) -> String {
                 String::new()
             };
             out.push_str(&format!(
-                "    seg[{}]: {} word={:<6} penalty={:<5} script={:<6} {}{}{}{}{}{}\n",
+                "    seg[{}]: {} word={:<6} penalty={:<5} script={:<6} {}{}{}{}\n",
                 j,
                 padded,
                 seg.word_cost,
                 seg.segment_penalty,
                 seg.script_cost,
                 conn_label,
                 seg.connection_cost,
-                ni_str,
-                pronoun_str,
                 te_str,
                 single_char_str,
             ));

engine/crates/lex-core/src/converter/reranker.rs

@@ -9,25 +9,6 @@ use crate::user_history::UserHistory;
 use super::cost::{conn_cost, script_cost};
 use super::viterbi::{RichSegment, ScoredPath};
 
-/// Non-independent kanji penalty for a segment.
-/// Returns penalty (> 0) if the segment is non-independent (形式名詞/補助動詞) with kanji surface.
-pub(super) fn non_independent_kanji_penalty(seg: &RichSegment, conn: &ConnectionMatrix) -> i64 {
-    if conn.is_non_independent(seg.left_id) && seg.surface.chars().any(is_kanji) {
-        settings().reranker.non_independent_kanji_penalty
-    } else {
-        0
-    }
-}
-
-/// Pronoun cost bonus for a segment (positive value = cost reduction).
-pub(super) fn pronoun_bonus(seg: &RichSegment, conn: &ConnectionMatrix) -> i64 {
-    if conn.is_pronoun(seg.left_id) {
-        settings().reranker.pronoun_cost_bonus
-    } else {
-        0
-    }
-}
-
 /// Te-form kanji penalty for a segment that follows て/で.
 /// `prev` is the preceding segment (None for the first segment).
 pub(super) fn te_form_kanji_penalty(
@@ -46,16 +27,6 @@ pub(super) fn te_form_kanji_penalty(
     0
 }
 
-/// Person name penalty for a segment.
-/// Returns penalty (> 0) if the segment is a person name (人名: 一般/姓/名; role == 6).
-pub(super) fn person_name_penalty(seg: &RichSegment, conn: &ConnectionMatrix) -> i64 {
-    if conn.is_person_name(seg.left_id) {
-        settings().reranker.person_name_penalty
-    } else {
-        0
-    }
-}
-
 /// Single-char kanji content-word penalty with dictionary compound exemption.
 pub(super) fn single_char_kanji_penalty(
     seg: &RichSegment,
@@ -222,20 +193,16 @@ pub fn rerank(
             .sum();
         path.viterbi_cost += total_script;
 
-        // Per-segment penalties: non-independent kanji, pronoun bonus,
-        // te-form kanji, single-char kanji content-word.
+        // Per-segment penalties: te-form kanji, single-char kanji content-word.
         if let Some(conn) = conn {
             for (i, seg) in path.segments.iter().enumerate() {
                 let prev = if i > 0 {
                     Some(&path.segments[i - 1])
                 } else {
                     None
                 };
-                path.viterbi_cost += non_independent_kanji_penalty(seg, conn);
-                path.viterbi_cost -= pronoun_bonus(seg, conn);
                 path.viterbi_cost += te_form_kanji_penalty(prev, seg, conn);
                 path.viterbi_cost += single_char_kanji_penalty(seg, i, &path.segments, conn, dict);
-                path.viterbi_cost += person_name_penalty(seg, conn);
             }
         }
     }
@@ -315,27 +282,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn non_independent_kanji_penalty_applied() {
-        // ID 2 = non-independent (role 4), ID 1 = content word (role 0)
-        let roles = vec![0u8, 0, 4];
-        let conn = conn_with_roles(roles);
-
-        // Path A: こと (hiragana, non-independent) — no penalty
-        // Path B: 事 (kanji, non-independent) — penalty applied
-        let mut paths = vec![
-            path(vec![seg("こと", "事", 2)], 100),
-            path(vec![seg("こと", "こと", 2)], 100),
-        ];
-
-        rerank(&mut paths, Some(&conn), None);
-
-        // The hiragana path should rank higher (lower cost)
-        assert_eq!(paths[0].segments[0].surface, "こと");
-        assert_eq!(paths[1].segments[0].surface, "事");
-        assert!(paths[0].viterbi_cost < paths[1].viterbi_cost);
-    }
-
     /// Build a minimal ConnectionMatrix with the given roles vector and
     /// function-word ID range.
     fn conn_with_roles_and_fw(roles: Vec<u8>, fw_min: u16, fw_max: u16) -> ConnectionMatrix {
@@ -344,30 +290,6 @@ mod tests {
         ConnectionMatrix::new_owned(num_ids, fw_min, fw_max, roles, costs)
     }
 
-    #[test]
-    fn non_independent_kanji_penalty_not_applied_to_content_words() {
-        // ID 1 = content word (role 0)
-        let roles = vec![0u8, 0];
-        let conn = conn_with_roles(roles);
-
-        // Both paths use content word IDs — no non-independent penalty
-        let mut paths = vec![
-            path(vec![seg("こと", "事", 1)], 100),
-            path(vec![seg("こと", "こと", 1)], 100),
-        ];
-
-        rerank(&mut paths, Some(&conn), None);
-
-        // Costs should differ only by script cost, not by non-independent penalty
-        let penalty = settings().reranker.non_independent_kanji_penalty;
-        let cost_diff = (paths[1].viterbi_cost - paths[0].viterbi_cost).abs();
-        assert!(
-            cost_diff < penalty,
-            "no non-independent penalty should be applied: diff = {}",
-            cost_diff
-        );
-    }
-
     #[test]
     fn te_form_kanji_penalty_applied() {
         // ID 2 = function word (fw_min=2, fw_max=2), ID 1 = content word
@@ -472,35 +394,6 @@ mod tests {
         assert!(paths[0].viterbi_cost < paths[1].viterbi_cost);
     }
 
-    #[test]
-    fn pronoun_bonus_applied() {
-        // ID 2 = pronoun (role 5), ID 1 = content word (role 0)
-        let roles = vec![0u8, 0, 5];
-        let conn = conn_with_roles(roles);
-
-        // Both paths have the same surface (hiragana) to isolate pronoun bonus.
-        // Path A: pronoun POS (id=2) — bonus applied
-        // Path B: content word POS (id=1) — no bonus
-        let mut paths = vec![
-            path(vec![seg("どれ", "どれ", 2)], 1000),
-            path(vec![seg("どれ", "どれ", 1)], 1000),
-        ];
-
-        rerank(&mut paths, Some(&conn), None);
-
-        // The pronoun path should rank higher (lower cost) after bonus
-        assert_eq!(
-            paths[0].segments[0].left_id, 2,
-            "pronoun path should rank first"
-        );
-        let bonus = settings().reranker.pronoun_cost_bonus;
-        let diff = paths[1].viterbi_cost - paths[0].viterbi_cost;
-        assert_eq!(
-            diff, bonus,
-            "cost difference should equal pronoun_cost_bonus"
-        );
-    }
-
     /// A minimal dictionary for testing compound exemption.
     struct MockDict {
         entries: Vec<(String, Vec<DictEntry>)>,
@@ -720,35 +613,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn person_name_penalty_applied() {
-        // ID 2 = person name (role 6), ID 1 = content word (role 0)
-        let roles = vec![0u8, 0, 6];
-        let conn = conn_with_roles(roles);
-
-        // Both paths have the same hiragana surface to isolate person name penalty.
-        // Path A: person name POS (id=2) — penalty applied
-        // Path B: content word POS (id=1) — no penalty
-        let mut paths = vec![
-            path(vec![seg("にしま", "にしま", 2)], 1000),
-            path(vec![seg("にしま", "にしま", 1)], 1000),
-        ];
-
-        rerank(&mut paths, Some(&conn), None);
-
-        // The content word path should rank higher (lower cost)
-        assert_eq!(
-            paths[0].segments[0].left_id, 1,
-            "content word path should rank first"
-        );
-        let penalty = settings().reranker.person_name_penalty;
-        let diff = paths[1].viterbi_cost - paths[0].viterbi_cost;
-        assert_eq!(
-            diff, penalty,
-            "cost difference should equal person_name_penalty"
-        );
-    }
-
     #[test]
     fn te_form_kanji_penalty_not_applied_to_non_te_function_word() {
         // ID 2 = function word (fw_min=2, fw_max=2), ID 1 = content word

engine/crates/lex-core/src/default_settings.toml

@@ -9,11 +9,8 @@ unknown_word_cost = 10000
 [reranker]
 length_variance_weight = 2000
 structure_cost_filter = 6000
-non_independent_kanji_penalty = 1500
 te_form_kanji_penalty = 3500
-pronoun_cost_bonus = 3500
 single_char_kanji_penalty = 4000
-person_name_penalty = 2000
 structure_cost_transition_cap = 5000
 
 [history]