Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions test_unstructured/staging/test_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -808,10 +808,10 @@ def test_elements_from_json_to_md_with_formula_fixture():
def test_elements_to_md_five_positional_args_order():
"""Lock (elements, filename, exclude_binary_image_data, encoding, normalize_formula)."""
m = mock_open()
elements = [Formula("x ∈ A")]
elements = [Formula("x × A")]
with patch("builtins.open", m):
out = base.elements_to_md(elements, "out.md", False, "latin-1", False)
assert out == "$$\nx ∈ A\n$$"
assert out == "$$\nx × A\n$$"
assert m.call_args.kwargs.get("encoding") == "latin-1"


Expand Down
65 changes: 65 additions & 0 deletions test_unstructured/staging/test_encoding_preflight.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
from functools import partial

import pytest

from unstructured.documents.elements import Text
from unstructured.staging.base import (
create_file_from_elements,
elements_to_json,
elements_to_md,
elements_to_ndjson,
elements_to_text,
)

EXPORTERS = [
elements_to_md,
elements_to_text,
elements_to_json,
elements_to_ndjson,
partial(create_file_from_elements, output_format="text"),
partial(create_file_from_elements, output_format="markdown"),
partial(create_file_from_elements, output_format="html"),
]


@pytest.mark.parametrize("export", EXPORTERS)
def test_unknown_encoding_does_not_truncate_an_existing_export(tmp_path, export):
destination = tmp_path / "document.txt"
destination.write_bytes(b"Existing document must survive")

with pytest.raises(LookupError):
export([Text("New document")], filename=str(destination), encoding="not-a-real-encoding")

assert destination.read_bytes() == b"Existing document must survive"


@pytest.mark.parametrize("export", [EXPORTERS[i] for i in (0, 1, 4, 5, 6)])
@pytest.mark.parametrize("already_exists", [False, True])
def test_unencodable_text_does_not_replace_or_create_an_export(tmp_path, export, already_exists):
destination = tmp_path / "document.txt"
if already_exists:
destination.write_bytes(b"Previous document")
# Put the failing code point beyond the first preflight block.
elements = [Text("A" * 70000 + "文")]

with pytest.raises(UnicodeEncodeError):
export(elements, filename=str(destination), encoding="ascii")

if already_exists:
assert destination.read_bytes() == b"Previous document"
else:
assert not destination.exists()


@pytest.mark.parametrize("export", EXPORTERS)
@pytest.mark.parametrize("encoding", ["utf-8", "utf-16", "iso2022_jp"])
def test_successful_encoding_preserves_export_content_and_codec_state(tmp_path, export, encoding):
destination = tmp_path / "document.txt"

elements = [Text("行一\n行二")]
expected = export(elements, encoding=encoding)
result = export(elements, filename=str(destination), encoding=encoding)

assert destination.read_text(encoding=encoding) == expected
if result is not None:
assert result == expected
33 changes: 22 additions & 11 deletions unstructured/staging/base.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import annotations

import base64
import codecs
import csv
import io
import json
Expand Down Expand Up @@ -461,6 +462,21 @@ def element_to_md(
return element.text


def _write_text_file(filename: str, text: str, encoding: str) -> None:
"""Check the entire encoding before opening (and truncating) a destination file.

Incremental preflight keeps extra encoded data bounded even for large exports. The final
text-mode write retains the existing newline and codec behavior. This protects against
encoding failures; operating-system write failures still propagate normally.
"""
encoder = codecs.getincrementalencoder(encoding)(errors="strict")
for start in range(0, len(text), 65536):
encoder.encode(text[start : start + 65536], final=False)
encoder.encode("", final=True)
with open(filename, "w", encoding=encoding) as file:
file.write(text)


def elements_to_md(
elements: Iterable[Element],
filename: Optional[str] = None,
Expand Down Expand Up @@ -501,8 +517,7 @@ def elements_to_md(
)

if filename is not None:
with open(filename, "w", encoding=encoding) as f:
f.write(markdown_content)
_write_text_file(filename, markdown_content, encoding)

return markdown_content

Expand Down Expand Up @@ -577,8 +592,7 @@ def create_file_from_elements(
no_group_by_page=no_group_by_page,
)
if filename is not None:
with open(filename, "w", encoding=encoding) as f:
f.write(content)
_write_text_file(filename, content, encoding)
return content
else:
# text: delegate write to elements_to_text when filename is set
Expand Down Expand Up @@ -606,8 +620,7 @@ def elements_to_json(
json_str = json.dumps(element_dicts, indent=indent, sort_keys=True)

if filename is not None:
with open(filename, "w", encoding=encoding) as f:
f.write(json_str)
_write_text_file(filename, json_str, encoding)

return json_str

Expand All @@ -629,8 +642,7 @@ def elements_to_ndjson(
ndjson_str = ndjson_dumps(element_dicts, sort_keys=True)

if filename is not None:
with open(filename, "w", encoding=encoding) as f:
f.write(ndjson_str)
_write_text_file(filename, ndjson_str, encoding)

return ndjson_str

Expand Down Expand Up @@ -718,9 +730,8 @@ def elements_to_text(
"""
element_cct = convert_to_text(elements)
if filename is not None:
with open(filename, "w", encoding=encoding) as f:
f.write(element_cct)
return None
_write_text_file(filename, element_cct, encoding)
return None
else:
return element_cct

Expand Down