fix(affixes): resolve destructive affix removal and protect root words (fixes #26)

CHANGELOG.md

@@ -6,6 +6,11 @@
 - Integration tests for CJS, ESM, and browser UMD artifacts.
 - Arabic NLP boundary and ReDoS resilience test suites.
 - Direct CDN links (jsdelivr & unpkg) and GitHub release bundle guide in documentation.
+- `AffixOptions` interface for configuring `minStemLength` and prefix/suffix stripping flags.
+
+### Fixed
+- Fixed destructive affix stripping in `removeArabicAffixes` by introducing minimum stem length constraint (`minStemLength >= 3`) to protect root letters, supporting compound prefixes (`بال`, `كال`, `ولل`), and preventing mangling of common words like "بيت" and "تفاح" (fixes [#26](https://github.com/Seen-Arabic/Arabic-Services-JavaScript/issues/26)).
+- Improved banned word detection in `tashfeerBannedWords` to verify both raw and stripped word similarity.
 
 ### Changed
 - Modularized monolithic `src/scripts/scripts.ts` into focused single-responsibility files (`tashkeel.ts`, `old-arabic.ts`, `tashfeer.ts`, `affixes.ts`, `letters.ts`) while keeping public exports and backward compatibility completely unchanged.

src/index.ts

@@ -19,3 +19,4 @@ export const ArabicServices = {
 export default ArabicServices;
 
 export type { OldArabicOptions };
+export type { AffixOptions } from './scripts/index.js';

src/scripts/affixes.ts

@@ -1,47 +1,134 @@
-import { ARABIC_PREFIXES, ARABIC_SUFFIXES } from '../constants/index.js';
-import { setCharAt } from '../utils/index.js';
+/**
+ * Options for configuring affix removal.
+ */
+export interface AffixOptions {
+	/**
+	 * Minimum length of the remaining stem after affix removal.
+	 * Words shorter than or equal to this length will not have affixes removed.
+	 * @default 3
+	 */
+	minStemLength?: number;
+	/**
+	 * Whether to remove common Arabic prefixes (e.g. "ال", "بال", "ولل", "و").
+	 * @default true
+	 */
+	prefixes?: boolean;
+	/**
+	 * Whether to remove common Arabic suffixes (e.g. "ون", "ين", "ات", "هم", "ة").
+	 * @default true
+	 */
+	suffixes?: boolean;
+}
+
+const COMPOUND_PREFIXES = ['كال', 'بال', 'فال', 'وال', 'ولل', 'فلل'];
+const DEFINITE_PREFIXES = ['ال', 'لل'];
+const CONJUNCTION_PREFIXES = ['و'];
+
+const LONG_SUFFIXES = ['تما', 'هما', 'كما'];
+const MEDIUM_SUFFIXES = ['هم', 'هن', 'كم', 'كن', 'نا', 'ها', 'ون', 'ين', 'ات', 'ية'];
+const SHORT_SUFFIXES = ['ة', 'ه', 'ي', 'ى'];
+
+/**
+ * Removes a matching prefix from a word if the remaining stem length satisfies minStemLength.
+ */
+function removePrefix(word: string, minStemLength: number): string {
+	for (const p of COMPOUND_PREFIXES) {
+		if (word.startsWith(p) && word.length - p.length >= minStemLength) {
+			return word.slice(p.length);
+		}
+	}
+	for (const p of DEFINITE_PREFIXES) {
+		if (word.startsWith(p) && word.length - p.length >= minStemLength) {
+			return word.slice(p.length);
+		}
+	}
+	for (const p of CONJUNCTION_PREFIXES) {
+		if (word.startsWith(p) && word.length - p.length >= minStemLength) {
+			return word.slice(p.length);
+		}
+	}
+	return word;
+}
 
 /**
- * Removes predefined affixes (prefixes and suffixes) from an Arabic word if it starts or ends with those affixes.
- * This function is designed specifically for processing Arabic text, where certain affixes might need to be removed
- * for linguistic, stylistic, or morphological reasons.
+ * Removes a matching suffix from a word if the remaining stem length satisfies minStemLength.
+ */
+function removeSuffix(word: string, minStemLength: number): string {
+	for (const s of LONG_SUFFIXES) {
+		if (word.endsWith(s) && word.length - s.length >= minStemLength) {
+			return word.slice(0, -s.length);
+		}
+	}
+	for (const s of MEDIUM_SUFFIXES) {
+		if (word.endsWith(s) && word.length - s.length >= minStemLength) {
+			return word.slice(0, -s.length);
+		}
+	}
+	for (const s of SHORT_SUFFIXES) {
+		if (word.endsWith(s) && word.length - s.length >= minStemLength) {
+			return word.slice(0, -s.length);
+		}
+	}
+	return word;
+}
+
+/**
+ * Removes predefined affixes (prefixes and suffixes) from an Arabic word while
+ * preserving a minimum stem length to prevent corrupting root letters.
  *
- * @param {string} word - The Arabic word from which the affixes are to be removed.
- * @returns {string} The word after removing any matching affixes. Returns the original word trimmed if no affix matches are found.
+ * @param {string} word - The Arabic word from which affixes are to be removed.
+ * @param {AffixOptions} [options] - Configuration options for affix removal.
+ * @returns {string} The word after removing matching affixes.
  */
-function removeArabicAffixesFromWord(word: string): string {
+function removeArabicAffixesFromWord(word: string, options: AffixOptions = {}): string {
+	const minStemLength = options.minStemLength ?? 3;
+	const stripPrefixes = options.prefixes ?? true;
+	const stripSuffixes = options.suffixes ?? true;
+
 	word = word.trim();
-	if (ARABIC_PREFIXES.includes(word.substring(0, 2))) {
-		// For: ALEF & LAM
-		word = setCharAt(word, 0, '');
-		word = setCharAt(word, 0, '');
-	} else if (ARABIC_PREFIXES.includes(word.substring(0, 1))) {
-		word = setCharAt(word, 0, '');
+	if (word.length <= minStemLength) {
+		return word;
 	}
 
-	if (ARABIC_SUFFIXES.includes(word.substring(word.length - 2))) {
-		word = setCharAt(word, word.length - 1, '');
-		word = setCharAt(word, word.length - 1, '');
-	} else if (ARABIC_SUFFIXES.includes(word[word.length - 1])) {
-		word = setCharAt(word, word.length - 1, '');
+	if (stripPrefixes) {
+		word = removePrefix(word, minStemLength);
+	}
+	if (stripSuffixes) {
+		word = removeSuffix(word, minStemLength);
 	}
-
 	return word;
 }
 
 /**
- * Removes predefined affixes (prefixes and suffixes) from an Arabic text if words start or end with those affixes.
- * This function is designed specifically for processing Arabic text, where certain affixes might need to be removed
- * for linguistic, stylistic, or morphological reasons.
+ * Removes predefined affixes (prefixes and suffixes) from words in an Arabic text while
+ * preserving minimum stem length to protect root consonants.
  *
- * @param {string} text - The Arabic text from which the affixes are to be removed.
- * @returns {string} The text after removing any matching affixes from each word. Returns the original text trimmed if no affix matches are found.
+ * @param {string} text - The Arabic text from which affixes are to be removed.
+ * @param {AffixOptions} [options] - Configuration options for affix removal.
+ * @returns {string} The text after removing matching affixes from each word.
+ *
+ * @example
+ * ```typescript
+ * removeArabicAffixes('المدرسون يخلصون لطلابهم');
+ * // returns 'مدرس يخلص طلاب'
+ * ```
  */
-export function removeArabicAffixes(text: string): string {
-	let new_sentence = '';
+export function removeArabicAffixes(text: string, options: AffixOptions = {}): string {
+	if (!text || typeof text !== 'string') {
+		return '';
+	}
 	text = text.trim();
-	for (const word of text.split(' ')) {
-		new_sentence += removeArabicAffixesFromWord(word) + ' ';
+	if (text.length === 0) {
+		return '';
+	}
+	const words = text.split(' ');
+	const processedWords: string[] = [];
+	for (const word of words) {
+		if (word === '') {
+			processedWords.push('');
+		} else {
+			processedWords.push(removeArabicAffixesFromWord(word, options));
+		}
 	}
-	return new_sentence.trim();
+	return processedWords.join(' ');
 }

src/scripts/tashfeer.ts

@@ -192,7 +192,9 @@ function bannedSimilarityRatio(string: string): number {
  */
 function checkIfBannedWord(string: string): boolean {
 	const std_ratio = 70;
-	return bannedSimilarityRatio(removeArabicAffixes(string)) >= std_ratio;
+	return (
+		bannedSimilarityRatio(string) >= std_ratio || bannedSimilarityRatio(removeArabicAffixes(string)) >= std_ratio
+	);
 }
 
 /**

tests/edge-cases.test.ts

@@ -32,12 +32,13 @@ describe('Arabic NLP Edge Cases & Robustness', () => {
 		});
 
 		it('should handle words that are shorter than standard affix length', () => {
-			// 'و' is not in ARABIC_PREFIXES so it stays unchanged
+			// 'و' is a single letter so it stays unchanged
 			expect(removeArabicAffixes('و')).toBe('و');
-			// 'ال' is an exact 2-letter prefix and is removed completely
-			expect(removeArabicAffixes('ال')).toBe('');
-			// single letter prefix 'ي'
-			expect(removeArabicAffixes('يقرأ')).toBe('قرأ');
+			// 'ال' is 2 letters; with default minStemLength=3 it is preserved, or stripped when minStemLength=0
+			expect(removeArabicAffixes('ال')).toBe('ال');
+			expect(removeArabicAffixes('ال', { minStemLength: 0 })).toBe('');
+			// Words with root letters are preserved
+			expect(removeArabicAffixes('يقرأ')).toBe('يقرأ');
 		});
 
 		it('should handle sentences with irregular spacing', () => {

tests/scripts.test.ts

@@ -182,52 +182,51 @@ describe('#wordToLetters', () => {
 });
 
 describe('#removeArabicAffixes', () => {
-	it('should remove "أ" prefix from a word', () => {
-		const word = 'أمل';
+	it('should remove "ال" prefix from a word', () => {
+		const word = 'الكتاب';
 		const result = ArabicServices.removeArabicAffixes(word);
-		expect(result).toEqual('مل');
+		expect(result).toEqual('كتاب');
 	});
 
-	it('should remove "ا" prefix & suffix "ة" from a word', () => {
-		const word = 'امرأة';
+	it('should remove compound prefix "بال" from a word', () => {
+		const word = 'بالكتاب';
 		const result = ArabicServices.removeArabicAffixes(word);
-		expect(result).toEqual('مرأ');
+		expect(result).toEqual('كتاب');
 	});
 
-	it('should remove "إ" prefix from a word', () => {
-		const word = 'إنسان';
+	it('should remove compound prefix "كال" from a word', () => {
+		const word = 'كالمطر';
 		const result = ArabicServices.removeArabicAffixes(word);
-		expect(result).toEqual('نسان');
+		expect(result).toEqual('مطر');
 	});
 
-	it('should remove "ال" prefix from a word', () => {
-		const word = 'الكتاب';
+	it('should remove compound prefix "ولل" from a word', () => {
+		const word = 'وللقدس';
 		const result = ArabicServices.removeArabicAffixes(word);
-		expect(result).toEqual('كتاب');
+		expect(result).toEqual('قدس');
 	});
 
-	it('should remove "ي" prefix from a word', () => {
-		const word = 'يوم';
+	it('should remove prefix "لل" from a word', () => {
+		const word = 'للمعلمين';
 		const result = ArabicServices.removeArabicAffixes(word);
-		expect(result).toEqual('وم');
+		expect(result).toEqual('معلم');
 	});
 
-	it('should remove "ت" prefix from a word', () => {
-		const word = 'تفاح';
+	it('should remove conjunction prefix "و" from a word', () => {
+		const word = 'وكتاب';
 		const result = ArabicServices.removeArabicAffixes(word);
-		expect(result).toEqual('فاح');
-	});
-
-	it('should remove "ن" prefix from a word', () => {
-		const word = 'نجم';
-		const result = ArabicServices.removeArabicAffixes(word);
-		expect(result).toEqual('جم');
+		expect(result).toEqual('كتاب');
 	});
 
-	it('should remove "ب" prefix from a word', () => {
-		const word = 'بيت';
-		const result = ArabicServices.removeArabicAffixes(word);
-		expect(result).toEqual('يت');
+	it('should preserve original root letters and not mangle non-affixed words', () => {
+		expect(ArabicServices.removeArabicAffixes('بيت')).toEqual('بيت');
+		expect(ArabicServices.removeArabicAffixes('تفاح')).toEqual('تفاح');
+		expect(ArabicServices.removeArabicAffixes('أمل')).toEqual('أمل');
+		expect(ArabicServices.removeArabicAffixes('نجم')).toEqual('نجم');
+		expect(ArabicServices.removeArabicAffixes('يوم')).toEqual('يوم');
+		expect(ArabicServices.removeArabicAffixes('إنسان')).toEqual('إنسان');
+		expect(ArabicServices.removeArabicAffixes('كبير')).toEqual('كبير');
+		expect(ArabicServices.removeArabicAffixes('ولد')).toEqual('ولد');
 	});
 
 	it('should remove "ة" suffix from a word', () => {
@@ -278,10 +277,34 @@ describe('#removeArabicAffixes', () => {
 		expect(result).toEqual('طلاب');
 	});
 
-	it('should remove prefix & suffix from a sentence', () => {
+	it('should remove "نا" suffix from a word', () => {
+		const word = 'كتابنا';
+		const result = ArabicServices.removeArabicAffixes(word);
+		expect(result).toEqual('كتاب');
+	});
+
+	it('should remove prefix & suffix from words in a sentence', () => {
 		const word = 'المدرسون يحبون طلابهم';
 		const result = ArabicServices.removeArabicAffixes(word);
-		expect(result).toEqual('مدرس حب طلاب');
+		expect(result).toEqual('مدرس يحب طلاب');
+	});
+
+	it('should remove "هما" suffix from a word', () => {
+		const word = 'كتابهما';
+		const result = ArabicServices.removeArabicAffixes(word);
+		expect(result).toEqual('كتاب');
+	});
+
+	it('should respect custom options when provided', () => {
+		const word = 'الكتاب';
+		expect(ArabicServices.removeArabicAffixes(word, { prefixes: false })).toEqual('الكتاب');
+		expect(ArabicServices.removeArabicAffixes('كتابنا', { suffixes: false })).toEqual('كتابنا');
+		expect(ArabicServices.removeArabicAffixes('كتابة', { minStemLength: 5 })).toEqual('كتابة');
+	});
+
+	it('should handle whitespace-only and invalid input', () => {
+		expect(ArabicServices.removeArabicAffixes('   ')).toEqual('');
+		expect(ArabicServices.removeArabicAffixes(null as unknown as string)).toEqual('');
 	});
 });