fix(pdf): normalize Arabic presentation forms (RTL phase 1)

packages/markitdown/src/markitdown/converters/_pdf_converter.py

@@ -1,6 +1,7 @@
 import sys
 import io
 import re
+import unicodedata
 from typing import BinaryIO, Any
 
 from .._base_converter import DocumentConverter, DocumentConverterResult
@@ -57,6 +58,32 @@ def _merge_partial_numbering_lines(text: str) -> str:
     return "\n".join(result_lines)
 
 
+# Arabic Presentation Forms (blocks U+FB50-U+FDFF and U+FE70-U+FEFF) that
+# some PDF producers emit instead of standard Arabic letters. They break
+# downstream text matching and LLM extraction, so they are normalized to
+# their standard forms. Only these two ranges are touched: the rest of the
+# text (including other compatibility characters) is left exactly as is.
+# See https://github.com/microsoft/markitdown/issues/2336.
+_PRESENTATION_FORM_RANGES = ((0xFB50, 0xFDFF), (0xFE70, 0xFEFF))
+
+
+def _normalize_arabic_presentation_forms(text: str) -> str:
+    """Replace Arabic presentation forms with standard Unicode letters.
+
+    Uses NFKC compatibility decomposition per character, restricted to the
+    presentation-form ranges, so ligatures (e.g. U+FEFB) also expand to
+    their letters. Idempotent on text without presentation forms.
+    """
+
+    def _normalize_char(char: str) -> str:
+        code = ord(char)
+        if any(start <= code <= end for start, end in _PRESENTATION_FORM_RANGES):
+            return unicodedata.normalize("NFKC", char)
+        return char
+
+    return "".join(_normalize_char(char) for char in text)
+
+
 # Load dependencies
 _dependency_exc_info = None
 try:
@@ -586,4 +613,7 @@ def convert(
         # Post-process to merge MasterFormat-style partial numbering with following text
         markdown = _merge_partial_numbering_lines(markdown)
 
+        # Post-process Arabic presentation forms into standard letters (#2336)
+        markdown = _normalize_arabic_presentation_forms(markdown)
+
         return DocumentConverterResult(markdown=markdown)

packages/markitdown/tests/test_pdf_rtl.py

@@ -0,0 +1,50 @@
+"""Arabic presentation-form normalization (microsoft/markitdown#2336).
+
+Every non-ASCII character below is a \\uXXXX escape: pasted
+presentation forms look identical to standard letters, so literals
+are banned in this file.
+"""
+
+from markitdown.converters._pdf_converter import (
+    _normalize_arabic_presentation_forms as normalize,
+)
+
+
+def _has_presentation_forms(text: str) -> bool:
+    return any(
+        0xFB50 <= ord(char) <= 0xFDFF or 0xFE70 <= ord(char) <= 0xFEFF
+        for char in text
+    )
+
+
+def test_forms_become_standard_letters() -> None:
+    # U+FE8D U+FEDF U+FEE3 U+FB8B U+FE94 U+FEEB
+    #   -> U+0627 U+0644 U+0645 U+0698 U+0629 U+0647
+    given = '\uFE8D\uFEDF\uFEE3\uFB8B\uFE94\uFEEB'
+    assert _has_presentation_forms(given)
+    assert normalize(given) == '\u0627\u0644\u0645\u0698\u0629\u0647'
+
+
+def test_ligature_expands_to_letters() -> None:
+    # U+FEFB -> U+0644 U+0627
+    assert normalize('\uFEFB') == '\u0644\u0627'
+
+
+def test_other_text_is_untouched() -> None:
+    # superscript-two, fi ligature (outside our ranges) and a standard
+    # Arabic word must pass through byte-identical.
+    text = 'Hello 604 / 2026 - \u00b2 \uFB01 \u0645\u0631\u062D\u0628\u0627'
+    assert normalize(text) == text
+
+
+def test_empty_and_plain_text_are_unchanged() -> None:
+    assert normalize('') == ''
+    assert normalize('Plain text 123.') == 'Plain text 123.'
+
+
+def test_normalization_is_idempotent() -> None:
+    once = normalize('\uFE8D report 604')
+    assert once == '\u0627 report 604'
+    assert normalize(once) == once
+    assert not _has_presentation_forms(once)
+