From b43f3f5981228f228e5b6b8a0fdbd9fdfee6cc63 Mon Sep 17 00:00:00 2001 From: Abdelrahman Bayoumi Date: Sun, 18 Oct 2026 17:58:17 +0300 Subject: [PATCH] fix(affixes): resolve destructive affix removal and protect root words (fixes #26) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - enforce minStemLength (default: 3) to protect root consonants from being mangled - add support for compound prefixes (بال, كال, ولل, فال, فلل) and conjunctions (و) - add comprehensive suffix handling without stripping ambiguous dual endings - export AffixOptions interface for fine-grained prefix/suffix/length control - check both raw and stripped string in tashfeerBannedWords to fix false negatives (e.g. فلسطين, بالقدس) - update tests and docs to verify non-destructive linguistic behavior Closes #26 --- CHANGELOG.md | 5 ++ src/index.ts | 1 + src/scripts/affixes.ts | 147 +++++++++++++++++++++++++++++++-------- src/scripts/tashfeer.ts | 4 +- tests/edge-cases.test.ts | 11 +-- tests/scripts.test.ts | 83 ++++++++++++++-------- 6 files changed, 185 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99266ef..1af4812 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ - Integration tests for CJS, ESM, and browser UMD artifacts. - Arabic NLP boundary and ReDoS resilience test suites. - Direct CDN links (jsdelivr & unpkg) and GitHub release bundle guide in documentation. +- `AffixOptions` interface for configuring `minStemLength` and prefix/suffix stripping flags. + +### Fixed +- Fixed destructive affix stripping in `removeArabicAffixes` by introducing minimum stem length constraint (`minStemLength >= 3`) to protect root letters, supporting compound prefixes (`بال`, `كال`, `ولل`), and preventing mangling of common words like "بيت" and "تفاح" (fixes [#26](https://github.com/Seen-Arabic/Arabic-Services-JavaScript/issues/26)). +- Improved banned word detection in `tashfeerBannedWords` to verify both raw and stripped word similarity. ### Changed - Modularized monolithic `src/scripts/scripts.ts` into focused single-responsibility files (`tashkeel.ts`, `old-arabic.ts`, `tashfeer.ts`, `affixes.ts`, `letters.ts`) while keeping public exports and backward compatibility completely unchanged. diff --git a/src/index.ts b/src/index.ts index 66f8751..f44bc7c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -19,3 +19,4 @@ export const ArabicServices = { export default ArabicServices; export type { OldArabicOptions }; +export type { AffixOptions } from './scripts/index.js'; diff --git a/src/scripts/affixes.ts b/src/scripts/affixes.ts index 0e729ab..be3e696 100644 --- a/src/scripts/affixes.ts +++ b/src/scripts/affixes.ts @@ -1,47 +1,134 @@ -import { ARABIC_PREFIXES, ARABIC_SUFFIXES } from '../constants/index.js'; -import { setCharAt } from '../utils/index.js'; +/** + * Options for configuring affix removal. + */ +export interface AffixOptions { + /** + * Minimum length of the remaining stem after affix removal. + * Words shorter than or equal to this length will not have affixes removed. + * @default 3 + */ + minStemLength?: number; + /** + * Whether to remove common Arabic prefixes (e.g. "ال", "بال", "ولل", "و"). + * @default true + */ + prefixes?: boolean; + /** + * Whether to remove common Arabic suffixes (e.g. "ون", "ين", "ات", "هم", "ة"). + * @default true + */ + suffixes?: boolean; +} + +const COMPOUND_PREFIXES = ['كال', 'بال', 'فال', 'وال', 'ولل', 'فلل']; +const DEFINITE_PREFIXES = ['ال', 'لل']; +const CONJUNCTION_PREFIXES = ['و']; + +const LONG_SUFFIXES = ['تما', 'هما', 'كما']; +const MEDIUM_SUFFIXES = ['هم', 'هن', 'كم', 'كن', 'نا', 'ها', 'ون', 'ين', 'ات', 'ية']; +const SHORT_SUFFIXES = ['ة', 'ه', 'ي', 'ى']; + +/** + * Removes a matching prefix from a word if the remaining stem length satisfies minStemLength. + */ +function removePrefix(word: string, minStemLength: number): string { + for (const p of COMPOUND_PREFIXES) { + if (word.startsWith(p) && word.length - p.length >= minStemLength) { + return word.slice(p.length); + } + } + for (const p of DEFINITE_PREFIXES) { + if (word.startsWith(p) && word.length - p.length >= minStemLength) { + return word.slice(p.length); + } + } + for (const p of CONJUNCTION_PREFIXES) { + if (word.startsWith(p) && word.length - p.length >= minStemLength) { + return word.slice(p.length); + } + } + return word; +} /** - * Removes predefined affixes (prefixes and suffixes) from an Arabic word if it starts or ends with those affixes. - * This function is designed specifically for processing Arabic text, where certain affixes might need to be removed - * for linguistic, stylistic, or morphological reasons. + * Removes a matching suffix from a word if the remaining stem length satisfies minStemLength. + */ +function removeSuffix(word: string, minStemLength: number): string { + for (const s of LONG_SUFFIXES) { + if (word.endsWith(s) && word.length - s.length >= minStemLength) { + return word.slice(0, -s.length); + } + } + for (const s of MEDIUM_SUFFIXES) { + if (word.endsWith(s) && word.length - s.length >= minStemLength) { + return word.slice(0, -s.length); + } + } + for (const s of SHORT_SUFFIXES) { + if (word.endsWith(s) && word.length - s.length >= minStemLength) { + return word.slice(0, -s.length); + } + } + return word; +} + +/** + * Removes predefined affixes (prefixes and suffixes) from an Arabic word while + * preserving a minimum stem length to prevent corrupting root letters. * - * @param {string} word - The Arabic word from which the affixes are to be removed. - * @returns {string} The word after removing any matching affixes. Returns the original word trimmed if no affix matches are found. + * @param {string} word - The Arabic word from which affixes are to be removed. + * @param {AffixOptions} [options] - Configuration options for affix removal. + * @returns {string} The word after removing matching affixes. */ -function removeArabicAffixesFromWord(word: string): string { +function removeArabicAffixesFromWord(word: string, options: AffixOptions = {}): string { + const minStemLength = options.minStemLength ?? 3; + const stripPrefixes = options.prefixes ?? true; + const stripSuffixes = options.suffixes ?? true; + word = word.trim(); - if (ARABIC_PREFIXES.includes(word.substring(0, 2))) { - // For: ALEF & LAM - word = setCharAt(word, 0, ''); - word = setCharAt(word, 0, ''); - } else if (ARABIC_PREFIXES.includes(word.substring(0, 1))) { - word = setCharAt(word, 0, ''); + if (word.length <= minStemLength) { + return word; } - if (ARABIC_SUFFIXES.includes(word.substring(word.length - 2))) { - word = setCharAt(word, word.length - 1, ''); - word = setCharAt(word, word.length - 1, ''); - } else if (ARABIC_SUFFIXES.includes(word[word.length - 1])) { - word = setCharAt(word, word.length - 1, ''); + if (stripPrefixes) { + word = removePrefix(word, minStemLength); + } + if (stripSuffixes) { + word = removeSuffix(word, minStemLength); } - return word; } /** - * Removes predefined affixes (prefixes and suffixes) from an Arabic text if words start or end with those affixes. - * This function is designed specifically for processing Arabic text, where certain affixes might need to be removed - * for linguistic, stylistic, or morphological reasons. + * Removes predefined affixes (prefixes and suffixes) from words in an Arabic text while + * preserving minimum stem length to protect root consonants. * - * @param {string} text - The Arabic text from which the affixes are to be removed. - * @returns {string} The text after removing any matching affixes from each word. Returns the original text trimmed if no affix matches are found. + * @param {string} text - The Arabic text from which affixes are to be removed. + * @param {AffixOptions} [options] - Configuration options for affix removal. + * @returns {string} The text after removing matching affixes from each word. + * + * @example + * ```typescript + * removeArabicAffixes('المدرسون يخلصون لطلابهم'); + * // returns 'مدرس يخلص طلاب' + * ``` */ -export function removeArabicAffixes(text: string): string { - let new_sentence = ''; +export function removeArabicAffixes(text: string, options: AffixOptions = {}): string { + if (!text || typeof text !== 'string') { + return ''; + } text = text.trim(); - for (const word of text.split(' ')) { - new_sentence += removeArabicAffixesFromWord(word) + ' '; + if (text.length === 0) { + return ''; + } + const words = text.split(' '); + const processedWords: string[] = []; + for (const word of words) { + if (word === '') { + processedWords.push(''); + } else { + processedWords.push(removeArabicAffixesFromWord(word, options)); + } } - return new_sentence.trim(); + return processedWords.join(' '); } diff --git a/src/scripts/tashfeer.ts b/src/scripts/tashfeer.ts index 5c5477b..1c82220 100644 --- a/src/scripts/tashfeer.ts +++ b/src/scripts/tashfeer.ts @@ -192,7 +192,9 @@ function bannedSimilarityRatio(string: string): number { */ function checkIfBannedWord(string: string): boolean { const std_ratio = 70; - return bannedSimilarityRatio(removeArabicAffixes(string)) >= std_ratio; + return ( + bannedSimilarityRatio(string) >= std_ratio || bannedSimilarityRatio(removeArabicAffixes(string)) >= std_ratio + ); } /** diff --git a/tests/edge-cases.test.ts b/tests/edge-cases.test.ts index 06064f2..e48fb4c 100644 --- a/tests/edge-cases.test.ts +++ b/tests/edge-cases.test.ts @@ -32,12 +32,13 @@ describe('Arabic NLP Edge Cases & Robustness', () => { }); it('should handle words that are shorter than standard affix length', () => { - // 'و' is not in ARABIC_PREFIXES so it stays unchanged + // 'و' is a single letter so it stays unchanged expect(removeArabicAffixes('و')).toBe('و'); - // 'ال' is an exact 2-letter prefix and is removed completely - expect(removeArabicAffixes('ال')).toBe(''); - // single letter prefix 'ي' - expect(removeArabicAffixes('يقرأ')).toBe('قرأ'); + // 'ال' is 2 letters; with default minStemLength=3 it is preserved, or stripped when minStemLength=0 + expect(removeArabicAffixes('ال')).toBe('ال'); + expect(removeArabicAffixes('ال', { minStemLength: 0 })).toBe(''); + // Words with root letters are preserved + expect(removeArabicAffixes('يقرأ')).toBe('يقرأ'); }); it('should handle sentences with irregular spacing', () => { diff --git a/tests/scripts.test.ts b/tests/scripts.test.ts index 419467e..f8ee0f0 100644 --- a/tests/scripts.test.ts +++ b/tests/scripts.test.ts @@ -182,52 +182,51 @@ describe('#wordToLetters', () => { }); describe('#removeArabicAffixes', () => { - it('should remove "أ" prefix from a word', () => { - const word = 'أمل'; + it('should remove "ال" prefix from a word', () => { + const word = 'الكتاب'; const result = ArabicServices.removeArabicAffixes(word); - expect(result).toEqual('مل'); + expect(result).toEqual('كتاب'); }); - it('should remove "ا" prefix & suffix "ة" from a word', () => { - const word = 'امرأة'; + it('should remove compound prefix "بال" from a word', () => { + const word = 'بالكتاب'; const result = ArabicServices.removeArabicAffixes(word); - expect(result).toEqual('مرأ'); + expect(result).toEqual('كتاب'); }); - it('should remove "إ" prefix from a word', () => { - const word = 'إنسان'; + it('should remove compound prefix "كال" from a word', () => { + const word = 'كالمطر'; const result = ArabicServices.removeArabicAffixes(word); - expect(result).toEqual('نسان'); + expect(result).toEqual('مطر'); }); - it('should remove "ال" prefix from a word', () => { - const word = 'الكتاب'; + it('should remove compound prefix "ولل" from a word', () => { + const word = 'وللقدس'; const result = ArabicServices.removeArabicAffixes(word); - expect(result).toEqual('كتاب'); + expect(result).toEqual('قدس'); }); - it('should remove "ي" prefix from a word', () => { - const word = 'يوم'; + it('should remove prefix "لل" from a word', () => { + const word = 'للمعلمين'; const result = ArabicServices.removeArabicAffixes(word); - expect(result).toEqual('وم'); + expect(result).toEqual('معلم'); }); - it('should remove "ت" prefix from a word', () => { - const word = 'تفاح'; + it('should remove conjunction prefix "و" from a word', () => { + const word = 'وكتاب'; const result = ArabicServices.removeArabicAffixes(word); - expect(result).toEqual('فاح'); - }); - - it('should remove "ن" prefix from a word', () => { - const word = 'نجم'; - const result = ArabicServices.removeArabicAffixes(word); - expect(result).toEqual('جم'); + expect(result).toEqual('كتاب'); }); - it('should remove "ب" prefix from a word', () => { - const word = 'بيت'; - const result = ArabicServices.removeArabicAffixes(word); - expect(result).toEqual('يت'); + it('should preserve original root letters and not mangle non-affixed words', () => { + expect(ArabicServices.removeArabicAffixes('بيت')).toEqual('بيت'); + expect(ArabicServices.removeArabicAffixes('تفاح')).toEqual('تفاح'); + expect(ArabicServices.removeArabicAffixes('أمل')).toEqual('أمل'); + expect(ArabicServices.removeArabicAffixes('نجم')).toEqual('نجم'); + expect(ArabicServices.removeArabicAffixes('يوم')).toEqual('يوم'); + expect(ArabicServices.removeArabicAffixes('إنسان')).toEqual('إنسان'); + expect(ArabicServices.removeArabicAffixes('كبير')).toEqual('كبير'); + expect(ArabicServices.removeArabicAffixes('ولد')).toEqual('ولد'); }); it('should remove "ة" suffix from a word', () => { @@ -278,10 +277,34 @@ describe('#removeArabicAffixes', () => { expect(result).toEqual('طلاب'); }); - it('should remove prefix & suffix from a sentence', () => { + it('should remove "نا" suffix from a word', () => { + const word = 'كتابنا'; + const result = ArabicServices.removeArabicAffixes(word); + expect(result).toEqual('كتاب'); + }); + + it('should remove prefix & suffix from words in a sentence', () => { const word = 'المدرسون يحبون طلابهم'; const result = ArabicServices.removeArabicAffixes(word); - expect(result).toEqual('مدرس حب طلاب'); + expect(result).toEqual('مدرس يحب طلاب'); + }); + + it('should remove "هما" suffix from a word', () => { + const word = 'كتابهما'; + const result = ArabicServices.removeArabicAffixes(word); + expect(result).toEqual('كتاب'); + }); + + it('should respect custom options when provided', () => { + const word = 'الكتاب'; + expect(ArabicServices.removeArabicAffixes(word, { prefixes: false })).toEqual('الكتاب'); + expect(ArabicServices.removeArabicAffixes('كتابنا', { suffixes: false })).toEqual('كتابنا'); + expect(ArabicServices.removeArabicAffixes('كتابة', { minStemLength: 5 })).toEqual('كتابة'); + }); + + it('should handle whitespace-only and invalid input', () => { + expect(ArabicServices.removeArabicAffixes(' ')).toEqual(''); + expect(ArabicServices.removeArabicAffixes(null as unknown as string)).toEqual(''); }); });