diff --git a/AGENT.md b/AGENT.md index 050689a..fca7945 100644 --- a/AGENT.md +++ b/AGENT.md @@ -30,7 +30,12 @@ This repository contains `arabic-services`: a zero-runtime-dependency TypeScript - `tashkeel.ts`: Diacritics (Harakat, Tanween, Quranic sequential tanween). - `banned-words.ts`: Word lists used in obfuscation/tashfeer algorithms. - `src/scripts/`: - - `scripts.ts`: Implementation of text manipulation functions (Tashkeel removal, Tatweel removal, Tashfeer, Old Arabic conversion, Affixes removal, etc.). + - `tashkeel.ts`: Tashkeel and Tatweel removal (`removeTashkeel`, `removeTatweel`). + - `old-arabic.ts`: Dotless Old Arabic script transformation (`toOldArabic`). + - `tashfeer.ts`: Visual obfuscation and banned-word encoding (`tashfeer`, `tashfeerBannedWords`, `toOldArabicAndTashfeerBannedWords`). + - `affixes.ts`: Prefix and suffix stripping heuristic (`removeArabicAffixes`). + - `letters.ts`: Conversion of words to pronounced letter names (`wordToLetters`). + - `index.ts`: Barrel file re-exporting all script operations. - `src/utils/`: - `string.util.ts`: Internal string manipulation and regex utility functions. - `tests/`: @@ -50,7 +55,7 @@ This repository contains `arabic-services`: a zero-runtime-dependency TypeScript - **Unicode Code Point Integrity**: Arabic characters have unique Unicode points for letters, ligatures, Tatweel (`\u0640`), Tashkeel (`\u064B`-`\u0652`, `\u0670`, sequential tanween, etc.). Avoid naive character splitting that breaks multi-byte or combining marks; always respect Unicode boundaries. - **Regular Expressions & ReDoS**: Regular expressions must be tested against catastrophic backtracking (ReDoS). Prefer pre-compiled regexes or Set/Dictionary lookups for character replacement rather than dynamic nested patterns. - **Dual Runtime Support**: The core library runs in both Node.js and web browsers. Never import Node-specific built-ins (`fs`, `path`, `crypto`, `os`) inside `src/`. -- **Test-Driven Changes**: Every bug fix, edge case, or new feature in `src/scripts/scripts.ts` must be accompanied by unit tests in `tests/scripts.test.ts`. +- **Test-Driven Changes**: Every bug fix, edge case, or new feature in `src/scripts/` must be accompanied by unit tests in `tests/scripts.test.ts` or `tests/edge-cases.test.ts`. --- diff --git a/CHANGELOG.md b/CHANGELOG.md index d717f85..99266ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Direct CDN links (jsdelivr & unpkg) and GitHub release bundle guide in documentation. ### Changed +- Modularized monolithic `src/scripts/scripts.ts` into focused single-responsibility files (`tashkeel.ts`, `old-arabic.ts`, `tashfeer.ts`, `affixes.ts`, `letters.ts`) while keeping public exports and backward compatibility completely unchanged. - Relocated standalone browser demo to `examples/browser/index.html`. - Unified CI workflow across Node.js 22.x and 24.x with ESLint. diff --git a/src/scripts/affixes.ts b/src/scripts/affixes.ts new file mode 100644 index 0000000..0e729ab --- /dev/null +++ b/src/scripts/affixes.ts @@ -0,0 +1,47 @@ +import { ARABIC_PREFIXES, ARABIC_SUFFIXES } from '../constants/index.js'; +import { setCharAt } from '../utils/index.js'; + +/** + * Removes predefined affixes (prefixes and suffixes) from an Arabic word if it starts or ends with those affixes. + * This function is designed specifically for processing Arabic text, where certain affixes might need to be removed + * for linguistic, stylistic, or morphological reasons. + * + * @param {string} word - The Arabic word from which the affixes are to be removed. + * @returns {string} The word after removing any matching affixes. Returns the original word trimmed if no affix matches are found. + */ +function removeArabicAffixesFromWord(word: string): string { + word = word.trim(); + if (ARABIC_PREFIXES.includes(word.substring(0, 2))) { + // For: ALEF & LAM + word = setCharAt(word, 0, ''); + word = setCharAt(word, 0, ''); + } else if (ARABIC_PREFIXES.includes(word.substring(0, 1))) { + word = setCharAt(word, 0, ''); + } + + if (ARABIC_SUFFIXES.includes(word.substring(word.length - 2))) { + word = setCharAt(word, word.length - 1, ''); + word = setCharAt(word, word.length - 1, ''); + } else if (ARABIC_SUFFIXES.includes(word[word.length - 1])) { + word = setCharAt(word, word.length - 1, ''); + } + + return word; +} + +/** + * Removes predefined affixes (prefixes and suffixes) from an Arabic text if words start or end with those affixes. + * This function is designed specifically for processing Arabic text, where certain affixes might need to be removed + * for linguistic, stylistic, or morphological reasons. + * + * @param {string} text - The Arabic text from which the affixes are to be removed. + * @returns {string} The text after removing any matching affixes from each word. Returns the original text trimmed if no affix matches are found. + */ +export function removeArabicAffixes(text: string): string { + let new_sentence = ''; + text = text.trim(); + for (const word of text.split(' ')) { + new_sentence += removeArabicAffixesFromWord(word) + ' '; + } + return new_sentence.trim(); +} diff --git a/src/scripts/index.ts b/src/scripts/index.ts index bb1a05c..cd68b17 100644 --- a/src/scripts/index.ts +++ b/src/scripts/index.ts @@ -1 +1,5 @@ -export * from './scripts.js'; +export * from './affixes.js'; +export * from './letters.js'; +export * from './old-arabic.js'; +export * from './tashfeer.js'; +export * from './tashkeel.js'; diff --git a/src/scripts/letters.ts b/src/scripts/letters.ts new file mode 100644 index 0000000..2b90c20 --- /dev/null +++ b/src/scripts/letters.ts @@ -0,0 +1,31 @@ +import { PRONOUNCED_LETTERS } from '../constants/index.js'; + +/** + * Converts a word to its pronounced letter representations based on PRONOUNCED_LETTERS. + * @param {string} word - The word to convert. + * @returns {string} The word with pronounced letters separated by spaces. + */ +export function wordToLetters(word: string): string { + const trimmedWord = word.trim(); + let newWord = ''; + + // Loop through each character in the input word + for (let i = 0; i < trimmedWord.length; i++) { + const letter = trimmedWord[i]; + + // Check if the current letter has a pronunciation in PRONOUNCED_LETTERS + if (PRONOUNCED_LETTERS.hasOwnProperty(letter)) { + newWord += PRONOUNCED_LETTERS[letter]; + + // Add a space after the pronounced letter unless it's the last letter in the word + if (i !== trimmedWord.length - 1) { + newWord += ' '; + } + } else { + // If the letter is not in PRONOUNCED_LETTERS, keep it unchanged + newWord += letter; + } + } + + return newWord.trim(); +} diff --git a/src/scripts/old-arabic.ts b/src/scripts/old-arabic.ts new file mode 100644 index 0000000..dd205ad --- /dev/null +++ b/src/scripts/old-arabic.ts @@ -0,0 +1,50 @@ +import { ARABIC_DOTLESS_DICT } from '../constants/index.js'; +import { fillDefaultOptions, OldArabicOptions } from '../options.js'; +import { removeTashkeel } from './tashkeel.js'; + +/** + * Remove all dots & tashkeel from text + * @param sentence string to convert to old arabic + * @param option + * @returns string in old arabic + * @example + * Input: "الخَيْلُ وَاللّيْلُ وَالبَيْداءُ تَعرِفُني" + * Output: "الحىل واللىل والٮىدا ٮعرڡٮى" + */ +export function toOldArabic(sentence: string, option: OldArabicOptions = {}): string { + const { replaceMidNoonWithBah, replaceMidYahWithBah } = fillDefaultOptions(option); + sentence = removeTashkeel(sentence.trim()); + let newSentence = ''; + for (let letter = 0; letter < sentence.length; letter++) { + // if letter is not Arabic letter => append to newSentence + if (!ARABIC_DOTLESS_DICT.hasOwnProperty(sentence[letter])) { + newSentence += sentence[letter]; + } + // Handle 'ن' Issue: + // if 'ن' is not last character && next character is Arabic letter or 'ـ' + // then replace it with 'ب' corresponding dotless letter => 'ٮ' + else if ( + replaceMidNoonWithBah && + sentence[letter] == 'ن' && + letter + 1 < sentence.length && + (ARABIC_DOTLESS_DICT.hasOwnProperty(sentence[letter + 1]) || sentence[letter + 1] == 'ـ') + ) { + newSentence += ARABIC_DOTLESS_DICT['ب']; + } + // Handle 'ي' Issue + // if 'ي' is not last character && next character is Arabic letter or 'ـ' + // then replace it with 'ب' corresponding dotless letter => 'ٮ' + else if ( + replaceMidYahWithBah && + sentence[letter] == 'ي' && + letter + 1 < sentence.length && + (ARABIC_DOTLESS_DICT.hasOwnProperty(sentence[letter + 1]) || sentence[letter + 1] == 'ـ') + ) { + newSentence += ARABIC_DOTLESS_DICT['ب']; + } else { + // if letter is Arabic letter => append corresponding dotless letter to newSentence + newSentence += ARABIC_DOTLESS_DICT[sentence[letter]]; + } + } + return newSentence; +} diff --git a/src/scripts/scripts.ts b/src/scripts/tashfeer.ts similarity index 58% rename from src/scripts/scripts.ts rename to src/scripts/tashfeer.ts index a466041..5c5477b 100644 --- a/src/scripts/scripts.ts +++ b/src/scripts/tashfeer.ts @@ -1,180 +1,15 @@ -import { ARABIC_DOTLESS_DICT, BANNED_WORDS, TASHKEEL } from '../constants/index.js'; +import { BANNED_WORDS } from '../constants/index.js'; import { ALEF, ALONE_LETTERS, - ARABIC_PREFIXES, - ARABIC_SUFFIXES, LETTERS_TASHFEER_REPLACEMENT_DICT, - PRONOUNCED_LETTERS, STANDARD_LETTERS, WAW, YAA, } from '../constants/arabic-letters.js'; -import { fillDefaultOptions, type OldArabicOptions } from '../options.js'; -import { setCharAt, similarityScore } from '../utils/index.js'; - -/** - * Remove all tashkeel from text - * @param text string to remove tashkeel from - * @returns string without tashkeel - * @example - * Input: "الخَيْلُ وَاللّيْلُ وَالبَيْداءُ تَعرِفُني" - * Output: "الخيل والليل والبيداء تعرفني" - */ -export function removeTashkeel(text: string): string { - return text - .trim() - .replace(new RegExp('[' + TASHKEEL.join('') + ']', 'g'), '') - .replace(/ٱ/g, 'ا'); -} - -/** - * Remove all dots & tashkeel from text - * @param sentence string to convert to old arabic - * @param option - * @returns string in old arabic - * @example - * Input: "الخَيْلُ وَاللّيْلُ وَالبَيْداءُ تَعرِفُني" - * Output: "الحىل واللىل والٮىدا ٮعرڡٮى" - */ -export function toOldArabic(sentence: string, option: OldArabicOptions = {}): string { - const { replaceMidNoonWithBah, replaceMidYahWithBah } = fillDefaultOptions(option); - sentence = removeTashkeel(sentence.trim()); - let newSentence = ''; - for (let letter = 0; letter < sentence.length; letter++) { - // if letter is not Arabic letter => append to newSentence - if (!ARABIC_DOTLESS_DICT.hasOwnProperty(sentence[letter])) { - newSentence += sentence[letter]; - } - // Handle 'ن' Issue: - // if 'ن' is not last character && next character is Arabic letter or 'ـ' - // then replace it with 'ب' corresponding dotless letter => 'ٮ' - else if ( - replaceMidNoonWithBah && - sentence[letter] == 'ن' && - letter + 1 < sentence.length && - (ARABIC_DOTLESS_DICT.hasOwnProperty(sentence[letter + 1]) || sentence[letter + 1] == 'ـ') - ) { - newSentence += ARABIC_DOTLESS_DICT['ب']; - } - // Handle 'ي' Issue - // if 'ي' is not last character && next character is Arabic letter or 'ـ' - // then replace it with 'ب' corresponding dotless letter => 'ٮ' - else if ( - replaceMidYahWithBah && - sentence[letter] == 'ي' && - letter + 1 < sentence.length && - (ARABIC_DOTLESS_DICT.hasOwnProperty(sentence[letter + 1]) || sentence[letter + 1] == 'ـ') - ) { - newSentence += ARABIC_DOTLESS_DICT['ب']; - } else { - // if letter is Arabic letter => append corresponding dotless letter to newSentence - newSentence += ARABIC_DOTLESS_DICT[sentence[letter]]; - } - } - return newSentence; -} - -export function toOldArabicAndTashfeerBannedWords(sentence: string, levelOfTashfeer: number = 2): string { - validateTashfeerLevel(levelOfTashfeer); - let new_sentence = ''; - const words = sentence.trim().split(' '); - for (const word of words) { - if (checkIfBannedWord(word)) { - new_sentence += tashfeerHandler(word, levelOfTashfeer) + ' '; - } else { - new_sentence += toOldArabic(word) + ' '; - } - } - return new_sentence.trim(); -} - -/** - * Remove all tatweel from text - * @param text string to remove tatweel from - * @returns string without tatweel - * @example - * Input: "رائـــــــع" - * Output: "رائع" - */ -export function removeTatweel(text: string): string { - return text.trim().replace(/ـ/g, ''); -} - -/** - * Converts a word to its pronounced letter representations based on PRONOUNCED_LETTERS. - * @param {string} word - The word to convert. - * @returns {string} The word with pronounced letters separated by spaces. - */ -export function wordToLetters(word: string): string { - const trimmedWord = word.trim(); - let newWord = ''; - - // Loop through each character in the input word - for (let i = 0; i < trimmedWord.length; i++) { - const letter = trimmedWord[i]; - - // Check if the current letter has a pronunciation in PRONOUNCED_LETTERS - if (PRONOUNCED_LETTERS.hasOwnProperty(letter)) { - newWord += PRONOUNCED_LETTERS[letter]; - - // Add a space after the pronounced letter unless it's the last letter in the word - if (i !== trimmedWord.length - 1) { - newWord += ' '; - } - } else { - // If the letter is not in PRONOUNCED_LETTERS, keep it unchanged - newWord += letter; - } - } - - return newWord.trim(); -} - -/** - * Removes predefined affixes (prefixes and suffixes) from an Arabic word if it starts or ends with those affixes. - * This function is designed specifically for processing Arabic text, where certain affixes might need to be removed - * for linguistic, stylistic, or morphological reasons. - * - * @param {string} word - The Arabic word from which the affixes are to be removed. - * @returns {string} The word after removing any matching affixes. Returns the original word trimmed if no affix matches are found. - */ -function removeArabicAffixesFromWord(word: string): string { - word = word.trim(); - if (ARABIC_PREFIXES.includes(word.substring(0, 2))) { - // For: ALEF & LAM - word = setCharAt(word, 0, ''); - word = setCharAt(word, 0, ''); - } else if (ARABIC_PREFIXES.includes(word.substring(0, 1))) { - word = setCharAt(word, 0, ''); - } - - if (ARABIC_SUFFIXES.includes(word.substring(word.length - 2))) { - word = setCharAt(word, word.length - 1, ''); - word = setCharAt(word, word.length - 1, ''); - } else if (ARABIC_SUFFIXES.includes(word[word.length - 1])) { - word = setCharAt(word, word.length - 1, ''); - } - - return word; -} - -/** - * Removes predefined affixes (prefixes and suffixes) from an Arabic text if words start or end with those affixes. - * This function is designed specifically for processing Arabic text, where certain affixes might need to be removed - * for linguistic, stylistic, or morphological reasons. - * - * @param {string} text - The Arabic text from which the affixes are to be removed. - * @returns {string} The text after removing any matching affixes from each word. Returns the original text trimmed if no affix matches are found. - */ -export function removeArabicAffixes(text: string): string { - let new_sentence = ''; - text = text.trim(); - for (const word of text.split(' ')) { - new_sentence += removeArabicAffixesFromWord(word) + ' '; - } - return new_sentence.trim(); -} +import { similarityScore } from '../utils/index.js'; +import { removeArabicAffixes } from './affixes.js'; +import { toOldArabic } from './old-arabic.js'; /** * Validates the encryption level to guarantee bounded execution and prevent infinite loops. @@ -319,6 +154,7 @@ function tashfeerHandler(word: string, level: number = 0): string { /** * Performs tashfeer encryption on a given sentence. * @param {string} sentence - The input sentence to be encrypted. + * @param {number} [levelOfTashfeer=1] - The encryption level. * @returns {string} The encrypted sentence. */ export function tashfeer(sentence: string, levelOfTashfeer: number = 1): string { @@ -380,3 +216,23 @@ export function tashfeerBannedWords(sentence: string, levelOfTashfeer: number = } return new_sentence.trim(); } + +/** + * Converts a sentence to old Arabic script and applies tashfeer to banned words. + * @param {string} sentence - The input sentence. + * @param {number} [levelOfTashfeer=2] - The encryption level (default is 2). + * @returns {string} The processed sentence. + */ +export function toOldArabicAndTashfeerBannedWords(sentence: string, levelOfTashfeer: number = 2): string { + validateTashfeerLevel(levelOfTashfeer); + let new_sentence = ''; + const words = sentence.trim().split(' '); + for (const word of words) { + if (checkIfBannedWord(word)) { + new_sentence += tashfeerHandler(word, levelOfTashfeer) + ' '; + } else { + new_sentence += toOldArabic(word) + ' '; + } + } + return new_sentence.trim(); +} diff --git a/src/scripts/tashkeel.ts b/src/scripts/tashkeel.ts new file mode 100644 index 0000000..350d6be --- /dev/null +++ b/src/scripts/tashkeel.ts @@ -0,0 +1,30 @@ +import { TASHKEEL } from '../constants/index.js'; + +/** + * Remove all tashkeel from text + * @param text string to remove tashkeel from + * @returns string without tashkeel + * @example + * ```js + * removeTashkeel('الخَيْـلُ وَاللّيْـلُ وَالبَيْـداءُ تَعرِفُني'); // 'الخيـل والليـل والبيـداء تعرفني' + * ``` + */ +export function removeTashkeel(text: string): string { + let cleanText = text.trim(); + cleanText = cleanText.replace(new RegExp(`[${TASHKEEL.join('')}]`, 'g'), ''); + cleanText = cleanText.replace(/ٱ/g, 'ا'); + return cleanText; +} + +/** + * Remove all tatweel from text + * @param text string to remove tatweel from + * @returns string without tatweel + * @example + * ```js + * removeTatweel('الخَيْـــلُ'); // 'الخَيْلُ' + * ``` + */ +export function removeTatweel(text: string): string { + return text.trim().replace(/\u0640/g, ''); +} diff --git a/tsconfig.json b/tsconfig.json index 2b25b0d..b51edc7 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,7 @@ { "compilerOptions": { "target": "es5", + "lib": ["ES2020", "DOM"], "module": "CommonJS", "strict": true, "declaration": true,