From 1a17c3b9929184535e4af1d5697fd2e178e92803 Mon Sep 17 00:00:00 2001 From: Ravi Shankar <142860126+RSKSOFFICIAL@users.noreply.github.com> Date: Sat, 5 Sep 2026 22:16:49 +0530 Subject: [PATCH 01/18] fix: support large input texts with chunking Signed-off-by: RSKKSOFFICIAL --- config.json | 14 +++++- lib/Service.py | 129 ++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 128 insertions(+), 15 deletions(-) diff --git a/config.json b/config.json index d82d550..0ed0757 100644 --- a/config.json +++ b/config.json @@ -3,7 +3,12 @@ "__comment::idle_polling_interval": "The interval in seconds to check for new messages when the app has no tasks", "__comment::tokenizer_file": "The tokenizer file name inside the model directory (loader.model_path)", "__comment::loader": "CTranslate2 loader options, see https://opennmt.net/CTranslate2/python/ctranslate2.Translator.html#ctranslate2.Translator.__init__. Use 'model_path' key for local paths or 'model_name' key for models hosted on Hugging Face. Both can't be used at the same time.", - "__comment::inference": "CTranslate2 inference options, see the kwargs in https://opennmt.net/CTranslate2/python/ctranslate2.Translator.html#ctranslate2.Translator.translate_batch", + "__comment::inference": "CTranslate2 inference options, see the kwargs in https://opennmt.net/CTranslate2/python/ctranslate2.Translator.html#ctranslate2.Translator.translate_batch. Note: max_decoding_length is computed dynamically per-chunk from chunking.max_decoding_multiplier and is not configurable here.", + "__comment::chunking": "Options controlling how large inputs are split before translation", + "__comment::chunking::chunk_threshold": "Word count above which the input is split into chunks before translation", + "__comment::chunking::chunk_size": "Maximum number of words per chunk when splitting large inputs", + "__comment::chunking::min_repetition_penalty": "Lower bound for repetition_penalty applied per chunk; overrides inference.repetition_penalty if that value is lower, to suppress runaway output loops on dense scripts such as Devanagari", + "__comment::chunking::max_decoding_multiplier": "Output token cap per chunk as a multiple of input token count; increase if output languages expand significantly (e.g. French, Arabic); default 3", "__comment::changes_to_the_config": "the program needs to be restarted if you change this file since it is stored in memory on startup", "log_level": 20, "idle_polling_interval": 5, @@ -18,8 +23,13 @@ "max_batch_size": 8192, "sampling_temperature": 0.0001, "disable_unk": true, - "max_decoding_length": 10000, "repetition_penalty": 1.3, "patience": 1.5 + }, + "chunking": { + "chunk_threshold": 250, + "chunk_size": 80, + "min_repetition_penalty": 1.5, + "max_decoding_multiplier": 3 } } diff --git a/lib/Service.py b/lib/Service.py index 9cc4026..a494039 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -7,6 +7,7 @@ import json import logging import os +import re from copy import deepcopy from time import perf_counter from typing import TypedDict @@ -18,6 +19,9 @@ logger = logging.getLogger(os.environ["APP_ID"] + __name__) +# Languages that do not use spaces between words — join chunks without a space separator +_NO_SPACE_LANGUAGES = {"zh", "ja", "th", "my", "km", "lo", "bo"} + class ServiceException(Exception): pass @@ -78,26 +82,125 @@ def load_model(self): except Exception as e: raise ServiceException("Error loading the translation model") from e + def _chunk_text(self, text: str, max_words: int) -> list[str]: + """Split text into sentence-boundary chunks of at most max_words words. + + Uses a simple sentence-boundary regex that handles: + - Period / exclamation / question mark followed by whitespace or end-of-string + - Newlines (already collapsed to spaces by clean_text, so this is a safety net) + + For no-space languages the concept of "word" doesn't apply the same way, + but the sentence-boundary split still works because those languages use + punctuation (。！？) as sentence terminators. + """ + # Sentence-boundary split: keep the delimiter attached to the preceding sentence + sentences = re.split(r'(?<=[.!?。！？])\s+', text) + + chunks: list[str] = [] + current_words: list[str] = [] + current_count = 0 + + for sentence in sentences: + word_count = len(sentence.split()) + if word_count == 0: + continue + + # If adding this sentence would overflow the chunk, flush first + if current_count + word_count > max_words and current_words: + chunks.append(" ".join(current_words)) + current_words = [] + current_count = 0 + + # If a single sentence is longer than max_words on its own, hard-split it + if word_count > max_words: + words = sentence.split() + for i in range(0, len(words), max_words): + chunks.append(" ".join(words[i:i + max_words])) + continue + + current_words.append(sentence) + current_count += word_count + + if current_words: + chunks.append(" ".join(current_words)) + + return chunks if chunks else [text] + + def _join_chunks(self, chunks: list[str], target_language: str) -> str: + """Join translated chunks respecting language-specific rules. + + - No-space languages (zh, ja, th, …): join with empty string. + - All other languages: join in forward order. + + Chunks are always kept in their original document order regardless of + source/target writing direction. Each chunk is translated independently + by the model, which already produces output in the correct reading order + for the target language. Reversing the chunk list would scramble the + logical sequence of the document. + """ + # Strip leading/trailing whitespace from each chunk before joining + chunks = [c.strip() for c in chunks if c.strip()] + if not chunks: + return "" + + target_base = target_language.split("_")[0].lower() + + separator = "" if target_base in _NO_SPACE_LANGUAGES else " " + return separator.join(chunks) + def translate(self, data: TranslateRequest) -> str: logger.debug(f"translating text to: {data['target_language']}") try: start = perf_counter() - input_tokens = self.tokenizer.Encode( - f"<2{data['target_language']}> {clean_text(data['input'])}", - out_type=str, - ) - results = self.translator.translate_batch( - [input_tokens], - batch_type="tokens", - **self.config["inference"], - ) + cleaned = clean_text(data["input"]) + + chunking = self.config.get("chunking", {}) + chunk_threshold = chunking.get("chunk_threshold", 250) + chunk_size = chunking.get("chunk_size", 80) + min_repetition_penalty = chunking.get("min_repetition_penalty", 1.5) + max_decoding_multiplier = chunking.get("max_decoding_multiplier", 3) + + # Only chunk if the input exceeds the threshold + if len(cleaned.split()) > chunk_threshold: + chunks = self._chunk_text(cleaned, chunk_size) + else: + chunks = [cleaned] + + translated_chunks: list[str] = [] + for chunk in chunks: + input_tokens = self.tokenizer.Encode( + f"<2{data['target_language']}> {chunk}", + out_type=str, + ) + # Cap max_decoding_length proportionally to the input token count. + # This applies to both chunked and non-chunked inputs to prevent + # runaway repetition loops (e.g. Hindi/Devanagari producing endless + # '=' characters). The multiplier is configurable via + # chunking.max_decoding_multiplier (default 3). Languages that + # expand significantly (e.g. French ~1.3x, Devanagari ~1.5x) may + # need a higher value. Floor of 64 handles very short inputs. + chunk_max_decoding = max(len(input_tokens) * max_decoding_multiplier, 64) + inference_config = { + **self.config["inference"], + "max_decoding_length": chunk_max_decoding, + "repetition_penalty": max( + self.config["inference"].get("repetition_penalty", 1.0), min_repetition_penalty + ), + } + results = self.translator.translate_batch( + [input_tokens], + batch_type="tokens", + **inference_config, + ) + + if len(results) == 0 or len(results[0].hypotheses) == 0: + raise ServiceException("Empty result returned from translator") - if len(results) == 0 or len(results[0].hypotheses) == 0: - raise ServiceException("Empty result returned from translator") + # todo: handle multiple hypotheses + translated_chunks.append(self.tokenizer.Decode(results[0].hypotheses[0])) - # todo: handle multiple hypotheses - translation = self.tokenizer.Decode(results[0].hypotheses[0]) + translation = self._join_chunks(translated_chunks, data["target_language"]) elapsed = perf_counter() - start logger.info(f"time taken: {elapsed:.2f}s") except Exception as e: From ffd2e3ae57745d23787d7af7e32a0414a05b9e6b Mon Sep 17 00:00:00 2001 From: Ravi Shankar <142860126+RSKSOFFICIAL@users.noreply.github.com> Date: Sat, 5 Sep 2026 22:39:47 +0530 Subject: [PATCH 02/18] fix: Replaced single quotes with double quotes Signed-off-by: RSKKSOFFICIAL --- lib/Service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Service.py b/lib/Service.py index a494039..2f74737 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -94,7 +94,7 @@ def _chunk_text(self, text: str, max_words: int) -> list[str]: punctuation (。！？) as sentence terminators. """ # Sentence-boundary split: keep the delimiter attached to the preceding sentence - sentences = re.split(r'(?<=[.!?。！？])\s+', text) + sentences = re.split(r"(?<=[.!?。！？])\s+", text) chunks: list[str] = [] current_words: list[str] = [] From c135e8bb7ad34f568457705455f8dc16b8514c27 Mon Sep 17 00:00:00 2001 From: Ravi Shankar <142860126+RSKSOFFICIAL@users.noreply.github.com> Date: Sat, 5 Sep 2026 22:53:31 +0530 Subject: [PATCH 03/18] fix: resolve ruff linting errors in Service.py Signed-off-by: RSKKSOFFICIAL --- lib/Service.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/lib/Service.py b/lib/Service.py index 2f74737..5b22761 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -22,6 +22,7 @@ # Languages that do not use spaces between words — join chunks without a space separator _NO_SPACE_LANGUAGES = {"zh", "ja", "th", "my", "km", "lo", "bo"} + class ServiceException(Exception): pass @@ -86,15 +87,14 @@ def _chunk_text(self, text: str, max_words: int) -> list[str]: """Split text into sentence-boundary chunks of at most max_words words. Uses a simple sentence-boundary regex that handles: - - Period / exclamation / question mark followed by whitespace or end-of-string - - Newlines (already collapsed to spaces by clean_text, so this is a safety net) + - Period / exclamation / question mark followed by whitespace For no-space languages the concept of "word" doesn't apply the same way, but the sentence-boundary split still works because those languages use - punctuation (。！？) as sentence terminators. + CJK sentence-ending punctuation (U+3002, U+FF01, U+FF1F) as terminators. """ # Sentence-boundary split: keep the delimiter attached to the preceding sentence - sentences = re.split(r"(?<=[.!?。！？])\s+", text) + sentences = re.split(r"(?<=[.!?\u3002\uff01\uff1f])\s+", text) # noqa: RUF001 chunks: list[str] = [] current_words: list[str] = [] @@ -161,11 +161,7 @@ def translate(self, data: TranslateRequest) -> str: min_repetition_penalty = chunking.get("min_repetition_penalty", 1.5) max_decoding_multiplier = chunking.get("max_decoding_multiplier", 3) - # Only chunk if the input exceeds the threshold - if len(cleaned.split()) > chunk_threshold: - chunks = self._chunk_text(cleaned, chunk_size) - else: - chunks = [cleaned] + chunks = self._chunk_text(cleaned, chunk_size) if len(cleaned.split()) > chunk_threshold else [cleaned] translated_chunks: list[str] = [] for chunk in chunks: From e42be61c40a6ed83a4690a6ae0b0884a874d0452 Mon Sep 17 00:00:00 2001 From: RSKKSOFFICIAL Date: Thu, 17 Sep 2026 15:12:37 +0000 Subject: [PATCH 04/18] fix: improve chunk batching and language handling Signed-off-by: RSKKSOFFICIAL --- lib/Service.py | 134 ++++++++++++++++++++++++++++++------------------- 1 file changed, 83 insertions(+), 51 deletions(-) diff --git a/lib/Service.py b/lib/Service.py index 5b22761..400d3ec 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -20,7 +20,7 @@ logger = logging.getLogger(os.environ["APP_ID"] + __name__) # Languages that do not use spaces between words — join chunks without a space separator -_NO_SPACE_LANGUAGES = {"zh", "ja", "th", "my", "km", "lo", "bo"} +_NO_SPACE_LANGUAGES = {"zh", "yue", "ja", "th", "my", "km", "lo", "bo", "dz", "shn"} class ServiceException(Exception): @@ -83,46 +83,63 @@ def load_model(self): except Exception as e: raise ServiceException("Error loading the translation model") from e - def _chunk_text(self, text: str, max_words: int) -> list[str]: - """Split text into sentence-boundary chunks of at most max_words words. + def _chunk_text(self, text: str, max_words: int, source_language: str = "") -> list[str]: + """Split text into sentence-boundary chunks of at most max_words words (or characters + for no-space languages such as Chinese, Japanese, Thai, etc.). Uses a simple sentence-boundary regex that handles: - - Period / exclamation / question mark followed by whitespace + - Period / exclamation / question mark followed by whitespace (Latin scripts) + - CJK sentence-ending punctuation (U+3002, U+FF01, U+FF1F) without requiring + trailing whitespace, since CJK sentences run together - For no-space languages the concept of "word" doesn't apply the same way, - but the sentence-boundary split still works because those languages use - CJK sentence-ending punctuation (U+3002, U+FF01, U+FF1F) as terminators. + For no-space languages split() always returns a single token regardless of + length, so character count is used as the unit instead of word count. """ - # Sentence-boundary split: keep the delimiter attached to the preceding sentence - sentences = re.split(r"(?<=[.!?\u3002\uff01\uff1f])\s+", text) # noqa: RUF001 + source_base = source_language.split("_")[0].lower() if source_language else "" + is_no_space = source_base in _NO_SPACE_LANGUAGES + + # Sentence-boundary split: keep the delimiter attached to the preceding sentence. + # For no-space languages (CJK etc.) use \s* because sentences run together without + # whitespace. For all other languages use \s+ to avoid splitting on abbreviations, + # decimals, URLs, and other mid-word periods (e.g. "Dr.", "3.14", "U.S.A"). + if is_no_space: + sentences = re.split(r"(?<=[。！？\u3002\uff01\uff1f])\s*", text) # noqa: RUF001 + else: + sentences = re.split(r"(?<=[.!?])\s+", text) chunks: list[str] = [] - current_words: list[str] = [] + current_parts: list[str] = [] current_count = 0 + sep = "" if is_no_space else " " for sentence in sentences: - word_count = len(sentence.split()) - if word_count == 0: + # Count characters for no-space languages, words for everything else + unit_count = len(sentence) if is_no_space else len(sentence.split()) + if unit_count == 0: continue # If adding this sentence would overflow the chunk, flush first - if current_count + word_count > max_words and current_words: - chunks.append(" ".join(current_words)) - current_words = [] + if current_count + unit_count > max_words and current_parts: + chunks.append(sep.join(current_parts)) + current_parts = [] current_count = 0 # If a single sentence is longer than max_words on its own, hard-split it - if word_count > max_words: - words = sentence.split() - for i in range(0, len(words), max_words): - chunks.append(" ".join(words[i:i + max_words])) + if unit_count > max_words: + if is_no_space: + for i in range(0, len(sentence), max_words): + chunks.append(sentence[i:i + max_words]) + else: + words = sentence.split() + for i in range(0, len(words), max_words): + chunks.append(" ".join(words[i:i + max_words])) continue - current_words.append(sentence) - current_count += word_count + current_parts.append(sentence) + current_count += unit_count - if current_words: - chunks.append(" ".join(current_words)) + if current_parts: + chunks.append(sep.join(current_parts)) return chunks if chunks else [text] @@ -161,40 +178,55 @@ def translate(self, data: TranslateRequest) -> str: min_repetition_penalty = chunking.get("min_repetition_penalty", 1.5) max_decoding_multiplier = chunking.get("max_decoding_multiplier", 3) - chunks = self._chunk_text(cleaned, chunk_size) if len(cleaned.split()) > chunk_threshold else [cleaned] + source_base = data.get("origin_language", "").split("_")[0].lower() + is_no_space_source = source_base in _NO_SPACE_LANGUAGES - translated_chunks: list[str] = [] - for chunk in chunks: - input_tokens = self.tokenizer.Encode( + # For no-space languages (CJK, Thai, etc.) use character count as the unit; + # split() always returns 1 for these scripts regardless of actual length. + text_size = len(cleaned) if is_no_space_source else len(cleaned.split()) + chunks = ( + self._chunk_text(cleaned, chunk_size, data.get("origin_language", "")) + if text_size > chunk_threshold + else [cleaned] + ) + + # Encode all chunks up-front so we can submit them in a single batch. + all_input_tokens = [ + self.tokenizer.Encode( f"<2{data['target_language']}> {chunk}", out_type=str, ) - # Cap max_decoding_length proportionally to the input token count. - # This applies to both chunked and non-chunked inputs to prevent - # runaway repetition loops (e.g. Hindi/Devanagari producing endless - # '=' characters). The multiplier is configurable via - # chunking.max_decoding_multiplier (default 3). Languages that - # expand significantly (e.g. French ~1.3x, Devanagari ~1.5x) may - # need a higher value. Floor of 64 handles very short inputs. - chunk_max_decoding = max(len(input_tokens) * max_decoding_multiplier, 64) - inference_config = { - **self.config["inference"], - "max_decoding_length": chunk_max_decoding, - "repetition_penalty": max( - self.config["inference"].get("repetition_penalty", 1.0), min_repetition_penalty - ), - } - results = self.translator.translate_batch( - [input_tokens], - batch_type="tokens", - **inference_config, - ) + for chunk in chunks + ] + + # Cap max_decoding_length using the longest chunk in the batch. + # This prevents runaway repetition loops (e.g. Hindi/Devanagari producing + # endless '=' characters) while still covering all chunks in one pass. + # The multiplier is configurable via chunking.max_decoding_multiplier + # (default 3). Languages that expand significantly (e.g. French ~1.3x, + # Devanagari ~1.5x) may need a higher value. Floor of 64 handles very + # short inputs. + max_input_tokens = max(len(t) for t in all_input_tokens) + batch_max_decoding = max(max_input_tokens * max_decoding_multiplier, 64) + inference_config = { + **self.config["inference"], + "max_decoding_length": batch_max_decoding, + "repetition_penalty": max( + self.config["inference"].get("repetition_penalty", 1.0), min_repetition_penalty + ), + } + + results = self.translator.translate_batch( + all_input_tokens, + batch_type="tokens", + **inference_config, + ) - if len(results) == 0 or len(results[0].hypotheses) == 0: - raise ServiceException("Empty result returned from translator") + if len(results) != len(chunks) or any(len(r.hypotheses) == 0 for r in results): + raise ServiceException("Empty result returned from translator") - # todo: handle multiple hypotheses - translated_chunks.append(self.tokenizer.Decode(results[0].hypotheses[0])) + # todo: handle multiple hypotheses + translated_chunks = [self.tokenizer.Decode(r.hypotheses[0]) for r in results] translation = self._join_chunks(translated_chunks, data["target_language"]) elapsed = perf_counter() - start From 5b870eacdde23132e82e3b0ca5dd08bab6abe779 Mon Sep 17 00:00:00 2001 From: RSKKSOFFICIAL Date: Thu, 17 Sep 2026 15:20:26 +0000 Subject: [PATCH 05/18] fix: add blank line after docstring summary in _chunk_text Signed-off-by: RSKKSOFFICIAL --- lib/Service.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/Service.py b/lib/Service.py index 400d3ec..1b63bf3 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -84,8 +84,10 @@ def load_model(self): raise ServiceException("Error loading the translation model") from e def _chunk_text(self, text: str, max_words: int, source_language: str = "") -> list[str]: - """Split text into sentence-boundary chunks of at most max_words words (or characters - for no-space languages such as Chinese, Japanese, Thai, etc.). + """Split text into sentence-boundary chunks of at most max_words words. + + For no-space languages (Chinese, Japanese, Thai, etc.) character count is used + instead of word count. Uses a simple sentence-boundary regex that handles: - Period / exclamation / question mark followed by whitespace (Latin scripts) From 38021cfcfc79f71086ab4eb3094bc3fbcc1e6517 Mon Sep 17 00:00:00 2001 From: Ravi Shankar Kumar <154051646+RSKKSOFFICIAL@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:15:13 +0530 Subject: [PATCH 06/18] Apply suggestion from @kyteinsky Co-authored-by: Anupam Kumar Signed-off-by: Ravi Shankar Kumar <154051646+RSKKSOFFICIAL@users.noreply.github.com> --- lib/Service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Service.py b/lib/Service.py index 1b63bf3..ddf4f9f 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -91,8 +91,8 @@ def _chunk_text(self, text: str, max_words: int, source_language: str = "") -> l Uses a simple sentence-boundary regex that handles: - Period / exclamation / question mark followed by whitespace (Latin scripts) - - CJK sentence-ending punctuation (U+3002, U+FF01, U+FF1F) without requiring - trailing whitespace, since CJK sentences run together + - CJK (Chinese, Japanese and Korean-alike languages) sentence-ending punctuation (U+3002, U+FF01, U+FF1F) without requiring + trailing whitespace, since CJK sentences run together. For no-space languages split() always returns a single token regardless of length, so character count is used as the unit instead of word count. From 2b947779af132fbb2d9e84cc97203b13e32213d5 Mon Sep 17 00:00:00 2001 From: RSKKSOFFICIAL Date: Tue, 22 Sep 2026 03:47:17 +0000 Subject: [PATCH 07/18] refactor: improved chunk translation and used translate_iterable Signed-off-by: RSKKSOFFICIAL --- config.json | 14 +++---- lib/Service.py | 100 +++++++++++++++++++++++++++++-------------------- 2 files changed, 66 insertions(+), 48 deletions(-) diff --git a/config.json b/config.json index 0ed0757..b2e0b96 100644 --- a/config.json +++ b/config.json @@ -3,12 +3,11 @@ "__comment::idle_polling_interval": "The interval in seconds to check for new messages when the app has no tasks", "__comment::tokenizer_file": "The tokenizer file name inside the model directory (loader.model_path)", "__comment::loader": "CTranslate2 loader options, see https://opennmt.net/CTranslate2/python/ctranslate2.Translator.html#ctranslate2.Translator.__init__. Use 'model_path' key for local paths or 'model_name' key for models hosted on Hugging Face. Both can't be used at the same time.", - "__comment::inference": "CTranslate2 inference options, see the kwargs in https://opennmt.net/CTranslate2/python/ctranslate2.Translator.html#ctranslate2.Translator.translate_batch. Note: max_decoding_length is computed dynamically per-chunk from chunking.max_decoding_multiplier and is not configurable here.", + "__comment::inference": "CTranslate2 inference options, see the kwargs in https://opennmt.net/CTranslate2/python/ctranslate2.Translator.html#ctranslate2.Translator.translate_batch.", + "__comment::inference::max_decoding_length": "Maximum number of output tokens per chunk. The MADLAD-400 3B/7B models are trained around 256 output tokens; 256 is the CTranslate2 default and a reasonable upper bound per chunk.", "__comment::chunking": "Options controlling how large inputs are split before translation", - "__comment::chunking::chunk_threshold": "Word count above which the input is split into chunks before translation", - "__comment::chunking::chunk_size": "Maximum number of words per chunk when splitting large inputs", - "__comment::chunking::min_repetition_penalty": "Lower bound for repetition_penalty applied per chunk; overrides inference.repetition_penalty if that value is lower, to suppress runaway output loops on dense scripts such as Devanagari", - "__comment::chunking::max_decoding_multiplier": "Output token cap per chunk as a multiple of input token count; increase if output languages expand significantly (e.g. French, Arabic); default 3", + "__comment::chunking::chunk_threshold": "Word count (or character count for no-space scripts) above which the input is split into chunks before translation", + "__comment::chunking::chunk_size": "Maximum words (or characters for no-space scripts) per chunk when splitting large inputs", "__comment::changes_to_the_config": "the program needs to be restarted if you change this file since it is stored in memory on startup", "log_level": 20, "idle_polling_interval": 5, @@ -21,6 +20,7 @@ }, "inference": { "max_batch_size": 8192, + "max_decoding_length": 256, "sampling_temperature": 0.0001, "disable_unk": true, "repetition_penalty": 1.3, @@ -28,8 +28,6 @@ }, "chunking": { "chunk_threshold": 250, - "chunk_size": 80, - "min_repetition_penalty": 1.5, - "max_decoding_multiplier": 3 + "chunk_size": 80 } } diff --git a/lib/Service.py b/lib/Service.py index ddf4f9f..79c653f 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -23,6 +23,28 @@ _NO_SPACE_LANGUAGES = {"zh", "yue", "ja", "th", "my", "km", "lo", "bo", "dz", "shn"} +def _is_no_space_text(text: str) -> bool: + """Return True when the source text appears to use a no-space writing system. + + Instead of relying on the origin_language tag (which may be "Detect Language" + or absent), we inspect the text itself. Languages like Chinese, Japanese, Thai, + Burmese, and Khmer have very few or no ASCII/Unicode space characters, giving a + space-to-total-character ratio close to zero. Space-delimited languages (English, + German, Arabic, Persian, Hindi, etc.) consistently produce a ratio above 10%. + + A threshold of 5% is conservative enough to avoid false positives on short + punctuation-heavy snippets while correctly identifying dense scripts. + + Empty or whitespace-only strings return False (they will produce an empty + translation regardless of counting method). + """ + stripped = text.strip() + if not stripped: + return False + space_count = stripped.count(" ") + stripped.count("\t") + stripped.count("\n") + return (space_count / len(stripped)) < 0.05 + + class ServiceException(Exception): pass @@ -83,26 +105,28 @@ def load_model(self): except Exception as e: raise ServiceException("Error loading the translation model") from e - def _chunk_text(self, text: str, max_words: int, source_language: str = "") -> list[str]: - """Split text into sentence-boundary chunks of at most max_words words. + def _chunk_text(self, text: str, max_words: int, is_no_space: bool = False) -> list[str]: + """Split text into sentence-boundary chunks of at most max_words words (or chars). - For no-space languages (Chinese, Japanese, Thai, etc.) character count is used - instead of word count. + Args: + text: The text to split. + max_words: Maximum words per chunk for space-delimited text, or maximum + characters per chunk for no-space writing systems. + is_no_space: Whether the source text uses a no-space writing system + (Chinese, Japanese, Thai, etc.). When True, character count is used + as the unit instead of word count. This should be derived from the + actual source text, not from a language code. Uses a simple sentence-boundary regex that handles: - Period / exclamation / question mark followed by whitespace (Latin scripts) - - CJK (Chinese, Japanese and Korean-alike languages) sentence-ending punctuation (U+3002, U+FF01, U+FF1F) without requiring - trailing whitespace, since CJK sentences run together. + - CJK (Chinese, Japanese and Korean-alike languages) sentence-ending + punctuation (U+3002, U+FF01, U+FF1F) without requiring trailing whitespace, since CJK + sentences run together. - For no-space languages split() always returns a single token regardless of - length, so character count is used as the unit instead of word count. """ - source_base = source_language.split("_")[0].lower() if source_language else "" - is_no_space = source_base in _NO_SPACE_LANGUAGES - # Sentence-boundary split: keep the delimiter attached to the preceding sentence. - # For no-space languages (CJK etc.) use \s* because sentences run together without - # whitespace. For all other languages use \s+ to avoid splitting on abbreviations, + # For no-space text (CJK etc.) use \s* because sentences run together without + # whitespace. For all other text use \s+ to avoid splitting on abbreviations, # decimals, URLs, and other mid-word periods (e.g. "Dr.", "3.14", "U.S.A"). if is_no_space: sentences = re.split(r"(?<=[。！？\u3002\uff01\uff1f])\s*", text) # noqa: RUF001 @@ -177,22 +201,26 @@ def translate(self, data: TranslateRequest) -> str: chunking = self.config.get("chunking", {}) chunk_threshold = chunking.get("chunk_threshold", 250) chunk_size = chunking.get("chunk_size", 80) - min_repetition_penalty = chunking.get("min_repetition_penalty", 1.5) - max_decoding_multiplier = chunking.get("max_decoding_multiplier", 3) - source_base = data.get("origin_language", "").split("_")[0].lower() - is_no_space_source = source_base in _NO_SPACE_LANGUAGES + # Detect whether the source text uses a no-space writing system (e.g. Chinese, + # Japanese, Thai) by examining the actual text rather than origin_language. + # origin_language may be "Detect Language" or otherwise unavailable, so it is + # not a reliable signal. A space ratio below 5% indicates a no-space script; + # for space-delimited languages (English, German, Arabic, etc.) the ratio is + # typically 15-20%. + is_no_space_source = _is_no_space_text(cleaned) - # For no-space languages (CJK, Thai, etc.) use character count as the unit; - # split() always returns 1 for these scripts regardless of actual length. + # For no-space text use character count as the threshold unit; + # for space-delimited text use word count. text_size = len(cleaned) if is_no_space_source else len(cleaned.split()) chunks = ( - self._chunk_text(cleaned, chunk_size, data.get("origin_language", "")) + self._chunk_text(cleaned, chunk_size, is_no_space=is_no_space_source) if text_size > chunk_threshold else [cleaned] ) - # Encode all chunks up-front so we can submit them in a single batch. + # Tokenise every chunk prefixed with the target-language tag expected by + # the MADLAD-400 model (e.g. "<2de> "). all_input_tokens = [ self.tokenizer.Encode( f"<2{data['target_language']}> {chunk}", @@ -201,28 +229,20 @@ def translate(self, data: TranslateRequest) -> str: for chunk in chunks ] - # Cap max_decoding_length using the longest chunk in the batch. - # This prevents runaway repetition loops (e.g. Hindi/Devanagari producing - # endless '=' characters) while still covering all chunks in one pass. - # The multiplier is configurable via chunking.max_decoding_multiplier - # (default 3). Languages that expand significantly (e.g. French ~1.3x, - # Devanagari ~1.5x) may need a higher value. Floor of 64 handles very - # short inputs. - max_input_tokens = max(len(t) for t in all_input_tokens) - batch_max_decoding = max(max_input_tokens * max_decoding_multiplier, 64) - inference_config = { - **self.config["inference"], - "max_decoding_length": batch_max_decoding, - "repetition_penalty": max( - self.config["inference"].get("repetition_penalty", 1.0), min_repetition_penalty - ), - } - - results = self.translator.translate_batch( + # translate_iterable streams all chunk token sequences through a single + # coordinated set of translate_batch calls, enabling asynchronous prefetching + # and (where inter_threads > 1) parallel translation. It preserves input + # order: results are yielded in the same order as the source iterable. + inference_config = {k: v for k, v in self.config["inference"].items() + if k != "max_batch_size"} + max_batch_size = self.config["inference"].get("max_batch_size", 32) + + results = list(self.translator.translate_iterable( all_input_tokens, + max_batch_size=max_batch_size, batch_type="tokens", **inference_config, - ) + )) if len(results) != len(chunks) or any(len(r.hypotheses) == 0 for r in results): raise ServiceException("Empty result returned from translator") From 621322f59e44f44b3115b69c71af393e3ae9c7c9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 03:55:25 +0000 Subject: [PATCH 08/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- lib/Service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Service.py b/lib/Service.py index 79c653f..96f4528 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -120,7 +120,7 @@ def _chunk_text(self, text: str, max_words: int, is_no_space: bool = False) -> l Uses a simple sentence-boundary regex that handles: - Period / exclamation / question mark followed by whitespace (Latin scripts) - CJK (Chinese, Japanese and Korean-alike languages) sentence-ending - punctuation (U+3002, U+FF01, U+FF1F) without requiring trailing whitespace, since CJK + punctuation (U+3002, U+FF01, U+FF1F) without requiring trailing whitespace, since CJK sentences run together. """ From ef1af9ee1462c56c1aa92896f7c285a486ba7fe4 Mon Sep 17 00:00:00 2001 From: Ravi Shankar Kumar <154051646+RSKKSOFFICIAL@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:37:14 +0530 Subject: [PATCH 09/18] docs: clarify origin language Co-authored-by: Anupam Kumar Signed-off-by: Ravi Shankar Kumar <154051646+RSKKSOFFICIAL@users.noreply.github.com> --- lib/Service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Service.py b/lib/Service.py index 96f4528..139f9a0 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -26,8 +26,8 @@ def _is_no_space_text(text: str) -> bool: """Return True when the source text appears to use a no-space writing system. - Instead of relying on the origin_language tag (which may be "Detect Language" - or absent), we inspect the text itself. Languages like Chinese, Japanese, Thai, + Instead of relying on the origin_language (which is always "detect_language"), + we inspect the text itself. Languages like Chinese, Japanese, Thai, Burmese, and Khmer have very few or no ASCII/Unicode space characters, giving a space-to-total-character ratio close to zero. Space-delimited languages (English, German, Arabic, Persian, Hindi, etc.) consistently produce a ratio above 10%. From 371b61e3514806df939d47242d6d0541625e2f9d Mon Sep 17 00:00:00 2001 From: Ravi Shankar Kumar <154051646+RSKKSOFFICIAL@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:40:33 +0530 Subject: [PATCH 10/18] refactor: simplify no-space sentence splitting Co-authored-by: Anupam Kumar Signed-off-by: Ravi Shankar Kumar <154051646+RSKKSOFFICIAL@users.noreply.github.com> --- lib/Service.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/Service.py b/lib/Service.py index 139f9a0..de18d56 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -125,11 +125,11 @@ def _chunk_text(self, text: str, max_words: int, is_no_space: bool = False) -> l """ # Sentence-boundary split: keep the delimiter attached to the preceding sentence. - # For no-space text (CJK etc.) use \s* because sentences run together without - # whitespace. For all other text use \s+ to avoid splitting on abbreviations, - # decimals, URLs, and other mid-word periods (e.g. "Dr.", "3.14", "U.S.A"). + # For no-space text (CJK etc.) use `\s*` because sentences run together without + # whitespace. For all other text use `\s+`. if is_no_space: - sentences = re.split(r"(?<=[。！？\u3002\uff01\uff1f])\s*", text) # noqa: RUF001 + # split on special sentence boundaries and spaces if present + sentences = re.split(r"(?<=[\u3002\uff01\uff1f])\s*", text) else: sentences = re.split(r"(?<=[.!?])\s+", text) From c1346d7937c73a3b30b840b4fa5d1776caad5b00 Mon Sep 17 00:00:00 2001 From: Ravi Shankar Kumar <154051646+RSKKSOFFICIAL@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:43:56 +0530 Subject: [PATCH 11/18] docs: simplify chunk joining docstring Co-authored-by: Anupam Kumar Signed-off-by: Ravi Shankar Kumar <154051646+RSKKSOFFICIAL@users.noreply.github.com> --- lib/Service.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/Service.py b/lib/Service.py index de18d56..201fde2 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -172,9 +172,6 @@ def _chunk_text(self, text: str, max_words: int, is_no_space: bool = False) -> l def _join_chunks(self, chunks: list[str], target_language: str) -> str: """Join translated chunks respecting language-specific rules. - - No-space languages (zh, ja, th, …): join with empty string. - - All other languages: join in forward order. - Chunks are always kept in their original document order regardless of source/target writing direction. Each chunk is translated independently by the model, which already produces output in the correct reading order From b20e6724362e64aa41f26d0246c6af847045d558 Mon Sep 17 00:00:00 2001 From: Ravi Shankar Kumar <154051646+RSKKSOFFICIAL@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:44:36 +0530 Subject: [PATCH 12/18] docs: simplify docstring Co-authored-by: Anupam Kumar Signed-off-by: Ravi Shankar Kumar <154051646+RSKKSOFFICIAL@users.noreply.github.com> --- lib/Service.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/Service.py b/lib/Service.py index 201fde2..afc61a2 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -199,12 +199,6 @@ def translate(self, data: TranslateRequest) -> str: chunk_threshold = chunking.get("chunk_threshold", 250) chunk_size = chunking.get("chunk_size", 80) - # Detect whether the source text uses a no-space writing system (e.g. Chinese, - # Japanese, Thai) by examining the actual text rather than origin_language. - # origin_language may be "Detect Language" or otherwise unavailable, so it is - # not a reliable signal. A space ratio below 5% indicates a no-space script; - # for space-delimited languages (English, German, Arabic, etc.) the ratio is - # typically 15-20%. is_no_space_source = _is_no_space_text(cleaned) # For no-space text use character count as the threshold unit; From ace6ee5c7ab62eb8602da5b47b7c32f7b9fefd90 Mon Sep 17 00:00:00 2001 From: RSKKSOFFICIAL Date: Thu, 24 Sep 2026 17:06:17 +0000 Subject: [PATCH 13/18] Fix: Simplified translate_iterable call Signed-off-by: RSKKSOFFICIAL --- config.json | 6 +++--- lib/Service.py | 30 ++++++++++++++++-------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/config.json b/config.json index b2e0b96..25fe9b2 100644 --- a/config.json +++ b/config.json @@ -3,10 +3,10 @@ "__comment::idle_polling_interval": "The interval in seconds to check for new messages when the app has no tasks", "__comment::tokenizer_file": "The tokenizer file name inside the model directory (loader.model_path)", "__comment::loader": "CTranslate2 loader options, see https://opennmt.net/CTranslate2/python/ctranslate2.Translator.html#ctranslate2.Translator.__init__. Use 'model_path' key for local paths or 'model_name' key for models hosted on Hugging Face. Both can't be used at the same time.", - "__comment::inference": "CTranslate2 inference options, see the kwargs in https://opennmt.net/CTranslate2/python/ctranslate2.Translator.html#ctranslate2.Translator.translate_batch.", + "__comment::inference": "CTranslate2 inference options, see the kwargs in https://opennmt.net/CTranslate2/python/ctranslate2.Translator.html#ctranslate2.Translator.translate_iterable.", "__comment::inference::max_decoding_length": "Maximum number of output tokens per chunk. The MADLAD-400 3B/7B models are trained around 256 output tokens; 256 is the CTranslate2 default and a reasonable upper bound per chunk.", "__comment::chunking": "Options controlling how large inputs are split before translation", - "__comment::chunking::chunk_threshold": "Word count (or character count for no-space scripts) above which the input is split into chunks before translation", + "__comment::chunking::chunk_threshold": "Token count above which the input is split into chunks before translation. Compared directly against the tokenizer output.", "__comment::chunking::chunk_size": "Maximum words (or characters for no-space scripts) per chunk when splitting large inputs", "__comment::changes_to_the_config": "the program needs to be restarted if you change this file since it is stored in memory on startup", "log_level": 20, @@ -27,7 +27,7 @@ "patience": 1.5 }, "chunking": { - "chunk_threshold": 250, + "chunk_threshold": 256, "chunk_size": 80 } } diff --git a/lib/Service.py b/lib/Service.py index afc61a2..41cc89a 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -127,11 +127,11 @@ def _chunk_text(self, text: str, max_words: int, is_no_space: bool = False) -> l # Sentence-boundary split: keep the delimiter attached to the preceding sentence. # For no-space text (CJK etc.) use `\s*` because sentences run together without # whitespace. For all other text use `\s+`. - if is_no_space: - # split on special sentence boundaries and spaces if present - sentences = re.split(r"(?<=[\u3002\uff01\uff1f])\s*", text) - else: - sentences = re.split(r"(?<=[.!?])\s+", text) + sentences = ( + re.split(r"(?<=[\u3002\uff01\uff1f])\s*", text) + if is_no_space + else re.split(r"(?<=[.!?])\s+", text) + ) chunks: list[str] = [] current_parts: list[str] = [] @@ -196,17 +196,22 @@ def translate(self, data: TranslateRequest) -> str: cleaned = clean_text(data["input"]) chunking = self.config.get("chunking", {}) - chunk_threshold = chunking.get("chunk_threshold", 250) + chunk_threshold = chunking.get("chunk_threshold", 256) chunk_size = chunking.get("chunk_size", 80) is_no_space_source = _is_no_space_text(cleaned) - # For no-space text use character count as the threshold unit; - # for space-delimited text use word count. - text_size = len(cleaned) if is_no_space_source else len(cleaned.split()) + # Use the tokenizer to estimate the input size in tokens so the threshold + # is directly comparable to max_decoding_length (which is also in tokens). + # This is accurate for every language without needing per-script heuristics: + # a 250-word English text and a 250-character CJK text both produce a token + # count that reflects the model's actual input length. + # The target-language prefix is excluded from the count since it is short + # and constant — we are sizing the source content only. + input_token_count = len(self.tokenizer.Encode(cleaned, out_type=str)) chunks = ( self._chunk_text(cleaned, chunk_size, is_no_space=is_no_space_source) - if text_size > chunk_threshold + if input_token_count > chunk_threshold else [cleaned] ) @@ -224,13 +229,10 @@ def translate(self, data: TranslateRequest) -> str: # coordinated set of translate_batch calls, enabling asynchronous prefetching # and (where inter_threads > 1) parallel translation. It preserves input # order: results are yielded in the same order as the source iterable. - inference_config = {k: v for k, v in self.config["inference"].items() - if k != "max_batch_size"} - max_batch_size = self.config["inference"].get("max_batch_size", 32) + inference_config = dict(self.config["inference"]) results = list(self.translator.translate_iterable( all_input_tokens, - max_batch_size=max_batch_size, batch_type="tokens", **inference_config, )) From 6b4b4a99e12a9dfbdf7d4b449795645a6e2a9413 Mon Sep 17 00:00:00 2001 From: Ravi Shankar <142860126+RSKSOFFICIAL@users.noreply.github.com> Date: Fri, 25 Sep 2026 18:45:34 +0530 Subject: [PATCH 14/18] docs: clarify chunking configuration Signed-off-by: Ravi Shankar <142860126+RSKSOFFICIAL@users.noreply.github.com> --- config.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/config.json b/config.json index 25fe9b2..2aad271 100644 --- a/config.json +++ b/config.json @@ -4,10 +4,7 @@ "__comment::tokenizer_file": "The tokenizer file name inside the model directory (loader.model_path)", "__comment::loader": "CTranslate2 loader options, see https://opennmt.net/CTranslate2/python/ctranslate2.Translator.html#ctranslate2.Translator.__init__. Use 'model_path' key for local paths or 'model_name' key for models hosted on Hugging Face. Both can't be used at the same time.", "__comment::inference": "CTranslate2 inference options, see the kwargs in https://opennmt.net/CTranslate2/python/ctranslate2.Translator.html#ctranslate2.Translator.translate_iterable.", - "__comment::inference::max_decoding_length": "Maximum number of output tokens per chunk. The MADLAD-400 3B/7B models are trained around 256 output tokens; 256 is the CTranslate2 default and a reasonable upper bound per chunk.", - "__comment::chunking": "Options controlling how large inputs are split before translation", - "__comment::chunking::chunk_threshold": "Token count above which the input is split into chunks before translation. Compared directly against the tokenizer output.", - "__comment::chunking::chunk_size": "Maximum words (or characters for no-space scripts) per chunk when splitting large inputs", + "__comment::chunking": "Text chunking options, including the token threshold and chunk size.", "__comment::changes_to_the_config": "the program needs to be restarted if you change this file since it is stored in memory on startup", "log_level": 20, "idle_polling_interval": 5, From 77651c47af6499643a8c314caf3e1412038abf19 Mon Sep 17 00:00:00 2001 From: Ravi Shankar <142860126+RSKSOFFICIAL@users.noreply.github.com> Date: Fri, 25 Sep 2026 19:19:01 +0530 Subject: [PATCH 15/18] docs: improved docstrings Signed-off-by: Ravi Shankar <142860126+RSKSOFFICIAL@users.noreply.github.com> --- lib/Service.py | 72 +++++++++++++------------------------------------- 1 file changed, 18 insertions(+), 54 deletions(-) diff --git a/lib/Service.py b/lib/Service.py index 41cc89a..68baa73 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -19,24 +19,20 @@ logger = logging.getLogger(os.environ["APP_ID"] + __name__) -# Languages that do not use spaces between words — join chunks without a space separator +# Languages that do not use spaces between words _NO_SPACE_LANGUAGES = {"zh", "yue", "ja", "th", "my", "km", "lo", "bo", "dz", "shn"} def _is_no_space_text(text: str) -> bool: """Return True when the source text appears to use a no-space writing system. - Instead of relying on the origin_language (which is always "detect_language"), - we inspect the text itself. Languages like Chinese, Japanese, Thai, - Burmese, and Khmer have very few or no ASCII/Unicode space characters, giving a - space-to-total-character ratio close to zero. Space-delimited languages (English, - German, Arabic, Persian, Hindi, etc.) consistently produce a ratio above 10%. - - A threshold of 5% is conservative enough to avoid false positives on short - punctuation-heavy snippets while correctly identifying dense scripts. - - Empty or whitespace-only strings return False (they will produce an empty - translation regardless of counting method). + Since the origin language is always "detect_language", + we inspect the text itself.A whitespace ratio below 5% is + treated as no-space text, which helps identify languages + such as Chinese, Japanese, Thai, and few more. + + Space-delimited languages (English,German, Arabic, Hindi, etc.) + consistently produce a ratio above 10%. """ stripped = text.strip() if not stripped: @@ -106,25 +102,15 @@ def load_model(self): raise ServiceException("Error loading the translation model") from e def _chunk_text(self, text: str, max_words: int, is_no_space: bool = False) -> list[str]: - """Split text into sentence-boundary chunks of at most max_words words (or chars). - - Args: - text: The text to split. - max_words: Maximum words per chunk for space-delimited text, or maximum - characters per chunk for no-space writing systems. - is_no_space: Whether the source text uses a no-space writing system - (Chinese, Japanese, Thai, etc.). When True, character count is used - as the unit instead of word count. This should be derived from the - actual source text, not from a language code. - - Uses a simple sentence-boundary regex that handles: - - Period / exclamation / question mark followed by whitespace (Latin scripts) - - CJK (Chinese, Japanese and Korean-alike languages) sentence-ending - punctuation (U+3002, U+FF01, U+FF1F) without requiring trailing whitespace, since CJK - sentences run together. + """Split text into sentence-boundary chunks of a maximum size. + + Space-delimited text is split by words, while no-space text is split by + characters. Sentence boundaries are preserved where possible, using + standard punctuation for space-delimited text and + CJK (Chinese, Japanese and Korean-alike languages) punctuation for no-space text. """ - # Sentence-boundary split: keep the delimiter attached to the preceding sentence. + # Keep sentence punctuation attached to the preceding sentence. # For no-space text (CJK etc.) use `\s*` because sentences run together without # whitespace. For all other text use `\s+`. sentences = ( @@ -139,18 +125,15 @@ def _chunk_text(self, text: str, max_words: int, is_no_space: bool = False) -> l sep = "" if is_no_space else " " for sentence in sentences: - # Count characters for no-space languages, words for everything else unit_count = len(sentence) if is_no_space else len(sentence.split()) if unit_count == 0: continue - # If adding this sentence would overflow the chunk, flush first if current_count + unit_count > max_words and current_parts: chunks.append(sep.join(current_parts)) current_parts = [] current_count = 0 - # If a single sentence is longer than max_words on its own, hard-split it if unit_count > max_words: if is_no_space: for i in range(0, len(sentence), max_words): @@ -172,13 +155,10 @@ def _chunk_text(self, text: str, max_words: int, is_no_space: bool = False) -> l def _join_chunks(self, chunks: list[str], target_language: str) -> str: """Join translated chunks respecting language-specific rules. - Chunks are always kept in their original document order regardless of - source/target writing direction. Each chunk is translated independently - by the model, which already produces output in the correct reading order - for the target language. Reversing the chunk list would scramble the - logical sequence of the document. + No-space languages are joined without a separator, while other languages + use a space. The translated chunks are already in the correct reading + order, so their order should not be reversed. """ - # Strip leading/trailing whitespace from each chunk before joining chunks = [c.strip() for c in chunks if c.strip()] if not chunks: return "" @@ -194,20 +174,10 @@ def translate(self, data: TranslateRequest) -> str: try: start = perf_counter() cleaned = clean_text(data["input"]) - chunking = self.config.get("chunking", {}) chunk_threshold = chunking.get("chunk_threshold", 256) chunk_size = chunking.get("chunk_size", 80) - is_no_space_source = _is_no_space_text(cleaned) - - # Use the tokenizer to estimate the input size in tokens so the threshold - # is directly comparable to max_decoding_length (which is also in tokens). - # This is accurate for every language without needing per-script heuristics: - # a 250-word English text and a 250-character CJK text both produce a token - # count that reflects the model's actual input length. - # The target-language prefix is excluded from the count since it is short - # and constant — we are sizing the source content only. input_token_count = len(self.tokenizer.Encode(cleaned, out_type=str)) chunks = ( self._chunk_text(cleaned, chunk_size, is_no_space=is_no_space_source) @@ -215,8 +185,6 @@ def translate(self, data: TranslateRequest) -> str: else [cleaned] ) - # Tokenise every chunk prefixed with the target-language tag expected by - # the MADLAD-400 model (e.g. "<2de> "). all_input_tokens = [ self.tokenizer.Encode( f"<2{data['target_language']}> {chunk}", @@ -225,10 +193,6 @@ def translate(self, data: TranslateRequest) -> str: for chunk in chunks ] - # translate_iterable streams all chunk token sequences through a single - # coordinated set of translate_batch calls, enabling asynchronous prefetching - # and (where inter_threads > 1) parallel translation. It preserves input - # order: results are yielded in the same order as the source iterable. inference_config = dict(self.config["inference"]) results = list(self.translator.translate_iterable( From 0c0fc6f49afd0ff73868471582c151a95fa21a2f Mon Sep 17 00:00:00 2001 From: Ravi Shankar <142860126+RSKSOFFICIAL@users.noreply.github.com> Date: Fri, 25 Sep 2026 19:22:30 +0530 Subject: [PATCH 16/18] docs: Fixed docstrings Signed-off-by: Ravi Shankar <142860126+RSKSOFFICIAL@users.noreply.github.com> --- lib/Service.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/Service.py b/lib/Service.py index 68baa73..e9e00d8 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -31,7 +31,7 @@ def _is_no_space_text(text: str) -> bool: treated as no-space text, which helps identify languages such as Chinese, Japanese, Thai, and few more. - Space-delimited languages (English,German, Arabic, Hindi, etc.) + Space-delimited languages (English, German, Arabic, Hindi, etc.) consistently produce a ratio above 10%. """ stripped = text.strip() @@ -104,11 +104,10 @@ def load_model(self): def _chunk_text(self, text: str, max_words: int, is_no_space: bool = False) -> list[str]: """Split text into sentence-boundary chunks of a maximum size. - Space-delimited text is split by words, while no-space text is split by characters. Sentence boundaries are preserved where possible, using standard punctuation for space-delimited text and - CJK (Chinese, Japanese and Korean-alike languages) punctuation for no-space text. + CJK (Chinese, Japanese, and Korean-alike languages) punctuation for no-space text. """ # Keep sentence punctuation attached to the preceding sentence. # For no-space text (CJK etc.) use `\s*` because sentences run together without From cbe1cbf9ba3b02f0e53cafccd0084895a80ab517 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 13:53:22 +0000 Subject: [PATCH 17/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- lib/Service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/Service.py b/lib/Service.py index e9e00d8..5f6e2f8 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -27,10 +27,10 @@ def _is_no_space_text(text: str) -> bool: """Return True when the source text appears to use a no-space writing system. Since the origin language is always "detect_language", - we inspect the text itself.A whitespace ratio below 5% is + we inspect the text itself.A whitespace ratio below 5% is treated as no-space text, which helps identify languages such as Chinese, Japanese, Thai, and few more. - + Space-delimited languages (English, German, Arabic, Hindi, etc.) consistently produce a ratio above 10%. """ @@ -106,7 +106,7 @@ def _chunk_text(self, text: str, max_words: int, is_no_space: bool = False) -> l Space-delimited text is split by words, while no-space text is split by characters. Sentence boundaries are preserved where possible, using - standard punctuation for space-delimited text and + standard punctuation for space-delimited text and CJK (Chinese, Japanese, and Korean-alike languages) punctuation for no-space text. """ # Keep sentence punctuation attached to the preceding sentence. From 42d7f1d7f6a0519d8cadf8862bb2a196eb0669ed Mon Sep 17 00:00:00 2001 From: Ravi Shankar <142860126+RSKSOFFICIAL@users.noreply.github.com> Date: Fri, 25 Sep 2026 19:50:45 +0530 Subject: [PATCH 18/18] refactor: rename max_words to max_units Signed-off-by: Ravi Shankar <142860126+RSKSOFFICIAL@users.noreply.github.com> --- lib/Service.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/lib/Service.py b/lib/Service.py index 5f6e2f8..681e650 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -27,12 +27,9 @@ def _is_no_space_text(text: str) -> bool: """Return True when the source text appears to use a no-space writing system. Since the origin language is always "detect_language", - we inspect the text itself.A whitespace ratio below 5% is + we inspect the text itself. A whitespace ratio below 5% is treated as no-space text, which helps identify languages such as Chinese, Japanese, Thai, and few more. - - Space-delimited languages (English, German, Arabic, Hindi, etc.) - consistently produce a ratio above 10%. """ stripped = text.strip() if not stripped: @@ -101,7 +98,7 @@ def load_model(self): except Exception as e: raise ServiceException("Error loading the translation model") from e - def _chunk_text(self, text: str, max_words: int, is_no_space: bool = False) -> list[str]: + def _chunk_text(self, text: str, max_units: int, is_no_space: bool = False) -> list[str]: """Split text into sentence-boundary chunks of a maximum size. Space-delimited text is split by words, while no-space text is split by @@ -128,19 +125,19 @@ def _chunk_text(self, text: str, max_words: int, is_no_space: bool = False) -> l if unit_count == 0: continue - if current_count + unit_count > max_words and current_parts: + if current_count + unit_count > max_units and current_parts: chunks.append(sep.join(current_parts)) current_parts = [] current_count = 0 - if unit_count > max_words: + if unit_count > max_units: if is_no_space: - for i in range(0, len(sentence), max_words): - chunks.append(sentence[i:i + max_words]) + for i in range(0, len(sentence), max_units): + chunks.append(sentence[i:i + max_units]) else: words = sentence.split() - for i in range(0, len(words), max_words): - chunks.append(" ".join(words[i:i + max_words])) + for i in range(0, len(words), max_units): + chunks.append(" ".join(words[i:i + max_units])) continue current_parts.append(sentence)