diff --git a/test_unstructured/staging/test_base.py b/test_unstructured/staging/test_base.py index a92fd4562e..602a2a8edc 100644 --- a/test_unstructured/staging/test_base.py +++ b/test_unstructured/staging/test_base.py @@ -808,10 +808,10 @@ def test_elements_from_json_to_md_with_formula_fixture(): def test_elements_to_md_five_positional_args_order(): """Lock (elements, filename, exclude_binary_image_data, encoding, normalize_formula).""" m = mock_open() - elements = [Formula("x ∈ A")] + elements = [Formula("x × A")] with patch("builtins.open", m): out = base.elements_to_md(elements, "out.md", False, "latin-1", False) - assert out == "$$\nx ∈ A\n$$" + assert out == "$$\nx × A\n$$" assert m.call_args.kwargs.get("encoding") == "latin-1" diff --git a/test_unstructured/staging/test_encoding_preflight.py b/test_unstructured/staging/test_encoding_preflight.py new file mode 100644 index 0000000000..5a2ae1b4ab --- /dev/null +++ b/test_unstructured/staging/test_encoding_preflight.py @@ -0,0 +1,65 @@ +from functools import partial + +import pytest + +from unstructured.documents.elements import Text +from unstructured.staging.base import ( + create_file_from_elements, + elements_to_json, + elements_to_md, + elements_to_ndjson, + elements_to_text, +) + +EXPORTERS = [ + elements_to_md, + elements_to_text, + elements_to_json, + elements_to_ndjson, + partial(create_file_from_elements, output_format="text"), + partial(create_file_from_elements, output_format="markdown"), + partial(create_file_from_elements, output_format="html"), +] + + +@pytest.mark.parametrize("export", EXPORTERS) +def test_unknown_encoding_does_not_truncate_an_existing_export(tmp_path, export): + destination = tmp_path / "document.txt" + destination.write_bytes(b"Existing document must survive") + + with pytest.raises(LookupError): + export([Text("New document")], filename=str(destination), encoding="not-a-real-encoding") + + assert destination.read_bytes() == b"Existing document must survive" + + +@pytest.mark.parametrize("export", [EXPORTERS[i] for i in (0, 1, 4, 5, 6)]) +@pytest.mark.parametrize("already_exists", [False, True]) +def test_unencodable_text_does_not_replace_or_create_an_export(tmp_path, export, already_exists): + destination = tmp_path / "document.txt" + if already_exists: + destination.write_bytes(b"Previous document") + # Put the failing code point beyond the first preflight block. + elements = [Text("A" * 70000 + "文")] + + with pytest.raises(UnicodeEncodeError): + export(elements, filename=str(destination), encoding="ascii") + + if already_exists: + assert destination.read_bytes() == b"Previous document" + else: + assert not destination.exists() + + +@pytest.mark.parametrize("export", EXPORTERS) +@pytest.mark.parametrize("encoding", ["utf-8", "utf-16", "iso2022_jp"]) +def test_successful_encoding_preserves_export_content_and_codec_state(tmp_path, export, encoding): + destination = tmp_path / "document.txt" + + elements = [Text("行一\n行二")] + expected = export(elements, encoding=encoding) + result = export(elements, filename=str(destination), encoding=encoding) + + assert destination.read_text(encoding=encoding) == expected + if result is not None: + assert result == expected diff --git a/unstructured/staging/base.py b/unstructured/staging/base.py index 65318620bc..d4571587ce 100644 --- a/unstructured/staging/base.py +++ b/unstructured/staging/base.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import codecs import csv import io import json @@ -461,6 +462,21 @@ def element_to_md( return element.text +def _write_text_file(filename: str, text: str, encoding: str) -> None: + """Check the entire encoding before opening (and truncating) a destination file. + + Incremental preflight keeps extra encoded data bounded even for large exports. The final + text-mode write retains the existing newline and codec behavior. This protects against + encoding failures; operating-system write failures still propagate normally. + """ + encoder = codecs.getincrementalencoder(encoding)(errors="strict") + for start in range(0, len(text), 65536): + encoder.encode(text[start : start + 65536], final=False) + encoder.encode("", final=True) + with open(filename, "w", encoding=encoding) as file: + file.write(text) + + def elements_to_md( elements: Iterable[Element], filename: Optional[str] = None, @@ -501,8 +517,7 @@ def elements_to_md( ) if filename is not None: - with open(filename, "w", encoding=encoding) as f: - f.write(markdown_content) + _write_text_file(filename, markdown_content, encoding) return markdown_content @@ -577,8 +592,7 @@ def create_file_from_elements( no_group_by_page=no_group_by_page, ) if filename is not None: - with open(filename, "w", encoding=encoding) as f: - f.write(content) + _write_text_file(filename, content, encoding) return content else: # text: delegate write to elements_to_text when filename is set @@ -606,8 +620,7 @@ def elements_to_json( json_str = json.dumps(element_dicts, indent=indent, sort_keys=True) if filename is not None: - with open(filename, "w", encoding=encoding) as f: - f.write(json_str) + _write_text_file(filename, json_str, encoding) return json_str @@ -629,8 +642,7 @@ def elements_to_ndjson( ndjson_str = ndjson_dumps(element_dicts, sort_keys=True) if filename is not None: - with open(filename, "w", encoding=encoding) as f: - f.write(ndjson_str) + _write_text_file(filename, ndjson_str, encoding) return ndjson_str @@ -718,9 +730,8 @@ def elements_to_text( """ element_cct = convert_to_text(elements) if filename is not None: - with open(filename, "w", encoding=encoding) as f: - f.write(element_cct) - return None + _write_text_file(filename, element_cct, encoding) + return None else: return element_cct