diff --git a/.github/workflows/extralit-server.build-docker-images.yml b/.github/workflows/extralit-server.build-docker-images.yml index 3ce6a58e1..ca3e3d458 100644 --- a/.github/workflows/extralit-server.build-docker-images.yml +++ b/.github/workflows/extralit-server.build-docker-images.yml @@ -125,6 +125,10 @@ jobs: run: | docker run --rm ${{ env.SERVER_DOCKER_IMAGE }}:${{ env.IMAGE_TAG }} python -m extralit_server start --help + - name: Check the retrieval payloads are baked into the image + run: | + docker run --rm ${{ env.SERVER_DOCKER_IMAGE }}:${{ env.IMAGE_TAG }} python smoke_retrieval_deps.py + - name: Push latest `extralit-server` image if: ${{ env.PUBLISH_LATEST == 'true' }} uses: docker/build-push-action@v5 diff --git a/CLAUDE.md b/CLAUDE.md index 3cbfc6d96..4ffdedd96 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,3 +62,6 @@ tag atomically. The version lives in three files — always change it with `python scripts/bump_version.py set --version X.Y.Z`, never by hand. See `docs/architecture/deployment.md` for the full pipeline + +## Gotchas & Rules +- **Check the library before writing a helper.** Before adding any function that renders, serializes, parses or walks a `DoclingDocument`, or splits chunks or fuses scores, check `docling_core.transforms.serializer.*` / `DocItem.export_to_*`, chonkie, and the DuckDB `lance` extension first. Phase 1 of retrieval shipped a `` serializer, a caption geometry join and a label→markdown CASE that docling already did; all were deleted. Functions with one consumer get inlined. diff --git a/extralit-hf-space b/extralit-hf-space index aa6077d8b..a05a1c01b 160000 --- a/extralit-hf-space +++ b/extralit-hf-space @@ -1 +1 @@ -Subproject commit aa6077d8b300cbeaca99482a67c01e4af9770b8e +Subproject commit a05a1c01b77303a69a9ccc34ef4665255af460b5 diff --git a/extralit-server/docker/server/.dockerignore b/extralit-server/docker/server/.dockerignore new file mode 100644 index 000000000..ee0676668 --- /dev/null +++ b/extralit-server/docker/server/.dockerignore @@ -0,0 +1,6 @@ +# Allowlist: the context is a wheel and two scripts, and should stay that way however much +# stray build output lands in this directory. +* +!scripts/*.py +!scripts/*.sh +!dist/*.whl diff --git a/extralit-server/docker/server/Dockerfile b/extralit-server/docker/server/Dockerfile index 0a508e4dd..2f640904d 100644 --- a/extralit-server/docker/server/Dockerfile +++ b/extralit-server/docker/server/Dockerfile @@ -1,25 +1,48 @@ +# The duckdb the venv resolves. Pinned so the 231 MB extension fetch can sit in a stage that does +# not depend on the application wheel; the `--check` below fails the build if the pin ever drifts. +ARG DUCKDB_VERSION=1.5.5 + +# Fetched in its own stage, ahead of anything that changes per build: DuckDB looks extensions up +# under a version-named directory, so this layer is identical until duckdb itself moves. +FROM python:3.12-slim AS lance +ARG DUCKDB_VERSION +COPY scripts/install_lance_extension.py /tmp/ +RUN python /tmp/install_lance_extension.py --duckdb-home /lance --duckdb-version "$DUCKDB_VERSION" + FROM python:3.12-slim AS builder -# Install uv -COPY --from=ghcr.io/astral-sh/uv:0.7.12 /uv /uvx /bin/ -# Copying extralit distribution files -COPY dist/*.whl /packages/ -RUN python -m venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" -ENV MAMBA_ROOT_PREFIX=/opt/venv -ENV CONDA_PREFIX=/opt/venv -ENV UV_CACHE_DIR=/home/extralit/.cache/uv +COPY --from=ghcr.io/astral-sh/uv:0.12.6 /uv /uvx /bin/ -RUN --mount=type=cache,target=/home/extralit/.cache/uv \ - apt-get update && \ +ENV VIRTUAL_ENV=/opt/venv \ + PATH="/opt/venv/bin:$PATH" \ + UV_CACHE_DIR=/root/.cache/uv \ + # Pay for the .pyc files once here instead of on every worker's first import. + UV_COMPILE_BYTECODE=1 \ + # The cache mount is a different filesystem from /opt/venv, so hardlinking cannot work. + UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=0 + +# psycopg2 is built from source. The toolchain stays in this stage — the runtime image copies +# nothing out of it but the venv, so there is nothing to purge afterwards. +RUN apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends libc6-dev libpq-dev gcc && \ - for wheel in /packages/*.whl; do uv pip install "$wheel"[postgresql]; done && \ - apt-get purge -y --auto-remove libc6-dev libpq-dev gcc && \ apt-get clean && \ - rm -rf /var/lib/apt/lists/* /packages + rm -rf /var/lib/apt/lists/* + +# --seed, though nothing here needs pip: without it `pip` falls through to the base image's +# /usr/local/bin/pip, which installs where /opt/venv/bin/python cannot see it. Dropping it saves +# 5 MB of 1847 and hands any derived image a silent wrong-interpreter install. +RUN uv venv --seed "$VIRTUAL_ENV" + +# The wheel is the only input that differs between two builds of the same tree, so it enters +# last — and as a bind mount, which leaves no copy of it behind in the stage. +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=dist,target=/packages \ + for wheel in /packages/*.whl; do uv pip install "$wheel"[postgresql]; done FROM python:3.12-slim +ARG DUCKDB_VERSION # Environment Variables ENV USERNAME="" @@ -39,24 +62,32 @@ RUN mkdir -p "$EXTRALIT_HOME_PATH" && \ chown extralit:extralit "$EXTRALIT_HOME_PATH" && \ apt-get update && \ apt-get upgrade -y && \ - apt-get install -y --no-install-recommends libpq-dev libgl1 libglib2.0-0 poppler-utils && \ + apt-get install -y --no-install-recommends libpq-dev libgl1 libglib2.0-0 poppler-utils tesseract-ocr-eng && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* VOLUME $EXTRALIT_HOME_PATH -COPY scripts/start_extralit_server.sh /home/extralit +# liteparse would otherwise download the language data on the first scanned page it meets. +ENV TESSDATA_PREFIX=/usr/share/tesseract-ocr/5/tessdata + +# Layers below are ordered by how often they change, because a rebuilt layer rebuilds every layer +# under it. DuckDB resolves extensions per user home, hence extralit's rather than root's. +COPY --from=lance --chown=extralit:extralit /lance /home/extralit/.duckdb +COPY --chmod=0755 scripts/start_extralit_server.sh scripts/smoke_retrieval_deps.py scripts/install_lance_extension.py /home/extralit/ # Destination folder must be the same as the builder one. Otherwise installed script won't work (since the installation fixes the path inside the script) COPY --chown=extralit:extralit --from=builder /opt/venv /opt/venv +ENV VIRTUAL_ENV=/opt/venv ENV PATH="/opt/venv/bin:$PATH" -ENV MAMBA_ROOT_PREFIX=/opt/venv -ENV CONDA_PREFIX=/opt/venv WORKDIR /home/extralit -RUN chmod +x start_extralit_server.sh USER extralit +# Fails the build rather than shipping an image whose first hybrid search cannot load the +# extension. Reads metadata only, so it survives the emulated arch of a release build. +RUN python install_lance_extension.py --check --duckdb-version "$DUCKDB_VERSION" + # Exposing ports EXPOSE 6900 diff --git a/extralit-server/docker/server/scripts/install_lance_extension.py b/extralit-server/docker/server/scripts/install_lance_extension.py new file mode 100644 index 000000000..65b02a5bc --- /dev/null +++ b/extralit-server/docker/server/scripts/install_lance_extension.py @@ -0,0 +1,124 @@ +"""Bake DuckDB's `lance` extension into the image, for this stage's target architecture. + + python install_lance_extension.py --duckdb-home /lance --duckdb-version 1.5.5 + python install_lance_extension.py --check --duckdb-version 1.5.5 + +`INSTALL lance` would be the obvious way to fetch it, but a release builds the non-native arch +under QEMU, where importing duckdb segfaults. Nothing here loads the native module: the payload +is a plain download and `--check` reads package metadata, so both survive emulation. + +The version is passed in rather than derived so the fetch can live in a stage that does not +depend on the application wheel — otherwise a one-line code change re-downloads 231 MB. `--check` +is what keeps that pin honest: it fails the build if the venv resolved a different duckdb. + +This extension is native code that the server later loads into its own process, so the payload +is fetched over TLS and checked against a pinned digest before it is written. An unpinned +version fails the build rather than trusting whatever the network returned. +""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import platform +import urllib.request +from importlib.metadata import version +from pathlib import Path + +REPOSITORY = "https://extensions.duckdb.org" +# The CDN 403s the default Python-urllib agent; any identifiable one is served. +USER_AGENT = "extralit-server image build" +PLATFORMS = {"aarch64": "linux_arm64", "x86_64": "linux_amd64"} + +#: sha256 of the *decompressed* extension, per duckdb version and target platform. Bumping +#: DUCKDB_VERSION means adding an entry here; see `--digest` for how to produce one. +DIGESTS = { + "1.5.5": { + "linux_amd64": "a8b1463e8541a960859b05c39096a60a3c777d12fd63d9867ce62bb251ab2a1a", + "linux_arm64": "a710b8c2453e996ff810c718ccc9b26253a1cf6c1b6687fad0dd805f8878e2c9", + }, +} + + +def target_platform() -> str: + machine = platform.machine() + try: + return PLATFORMS[machine] + except KeyError: + raise SystemExit(f"no DuckDB extension platform for {machine}") from None + + +def expected_digest(duckdb_version: str, target: str) -> str: + try: + return DIGESTS[duckdb_version][target] + except KeyError: + raise SystemExit( + f"no pinned sha256 for the lance extension at duckdb {duckdb_version} on {target}. " + f"This build will not load an unverified native extension into the server. Obtain the " + f"digest, confirm it independently of this download, and add it to DIGESTS in " + f"{Path(__file__).name} (`--digest` prints what the repository is currently serving)." + ) from None + + +def download(duckdb_version: str, target: str) -> bytes: + url = f"{REPOSITORY}/v{duckdb_version}/{target}/lance.duckdb_extension.gz" + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request) as response: + return gzip.decompress(response.read()) + + +def check(expected: str) -> None: + installed = version("duckdb") + if installed != expected: + raise SystemExit( + f"the image bakes the lance extension for duckdb {expected}, but the venv resolved " + f"{installed}. DuckDB looks extensions up under a version-named directory, so this " + f"would ship an image whose first hybrid search fails. Set DUCKDB_VERSION={installed} " + f"in the Dockerfile." + ) + + +def fetch(duckdb_version: str, duckdb_home: Path) -> None: + target = target_platform() + expected = expected_digest(duckdb_version, target) + + payload = download(duckdb_version, target) + digest = hashlib.sha256(payload).hexdigest() + if digest != expected: + # Nothing is written: the bytes are not what this image is pinned to. + raise SystemExit( + f"the lance extension served for duckdb {duckdb_version} on {target} has sha256 " + f"{digest}, but this image pins {expected}. Refusing to bake an unverified native " + f"extension." + ) + + destination = duckdb_home / "extensions" / f"v{duckdb_version}" / target + destination.mkdir(parents=True, exist_ok=True) + written = destination / "lance.duckdb_extension" + written.write_bytes(payload) + print(f"wrote {written} ({len(payload)} bytes, sha256 {digest})") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--duckdb-version", help="defaults to the installed duckdb") + parser.add_argument("--duckdb-home", type=Path, default=Path.home() / ".duckdb") + parser.add_argument("--check", action="store_true", help="assert the venv agrees with the pin") + parser.add_argument("--digest", action="store_true", help="print the served digest, for pinning") + args = parser.parse_args() + + duckdb_version = args.duckdb_version or version("duckdb") + if args.check: + check(duckdb_version) + print(f"ok: venv duckdb matches the baked lance extension ({duckdb_version})") + return + if args.digest: + target = target_platform() + print(f'"{target}": "{hashlib.sha256(download(duckdb_version, target)).hexdigest()}",') + return + fetch(duckdb_version, args.duckdb_home) + + +if __name__ == "__main__": + main() diff --git a/extralit-server/docker/server/scripts/smoke_retrieval_deps.py b/extralit-server/docker/server/scripts/smoke_retrieval_deps.py new file mode 100644 index 000000000..21a1d0c4c --- /dev/null +++ b/extralit-server/docker/server/scripts/smoke_retrieval_deps.py @@ -0,0 +1,73 @@ +"""Assert the retrieval pipeline's non-Python payloads are present in this environment. + + docker run --rm extralit/extralit-server:latest python smoke_retrieval_deps.py + +Each of these fails late and confusingly in production — a first hybrid search that cannot +reach the DuckDB extension repository, or a first scanned page that finds no language data — +so the image build runs this instead of discovering them under load. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + + +def check_lance_extension() -> str: + import duckdb + + connection = duckdb.connect() + # No INSTALL: the point is that the extension is already on disk. + connection.execute("LOAD lance") + functions = connection.execute( + "select function_name from duckdb_functions() where function_name in ('lance_fts', 'lance_vector_search')" + ).fetchall() + missing = {"lance_fts", "lance_vector_search"} - {name for (name,) in functions} + if missing: + raise RuntimeError(f"the lance extension loaded without {sorted(missing)}") + return f"duckdb {duckdb.__version__} + lance extension" + + +def check_liteparse() -> str: + import liteparse + + parser = liteparse.LiteParse(extract_blocks=True, quiet=True) + parser.close() + return f"liteparse {liteparse.__version__}" + + +def check_tessdata() -> str: + prefix = os.environ.get("TESSDATA_PREFIX") + if not prefix: + raise RuntimeError("TESSDATA_PREFIX is unset, so OCR would download language data on first use") + if not (Path(prefix) / "eng.traineddata").is_file(): + raise RuntimeError(f"no eng.traineddata under {prefix}") + return f"tessdata at {prefix}" + + +def check_chonkie() -> str: + from chonkie import RecursiveChunker, RecursiveRules + + chunker = RecursiveChunker(tokenizer="character", chunk_size=64, rules=RecursiveRules()) + if not chunker("one two three. four five six."): + raise RuntimeError("RecursiveChunker returned no chunks") + import chonkie + + return f"chonkie {chonkie.__version__}" + + +def main() -> int: + failures = 0 + for check in (check_lance_extension, check_liteparse, check_tessdata, check_chonkie): + name = check.__name__.removeprefix("check_") + try: + print(f"ok {name}: {check()}") + except Exception as error: + failures += 1 + print(f"FAIL {name}: {type(error).__name__}: {error}", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/extralit-server/pyproject.toml b/extralit-server/pyproject.toml index dcb48fd3d..29a1ea3c8 100644 --- a/extralit-server/pyproject.toml +++ b/extralit-server/pyproject.toml @@ -80,6 +80,9 @@ dependencies = [ "docling-core>=2.91.0,<3.0.0", "pyarrow>=23.0.1", "pdf-inspector>=1.14.2", + # abi3 wheels from 2.14, so the block extractor is not tied to one CPython + "liteparse>=2.14.0", + "chonkie>=1.7.0", "xxhash>=3.6.0", "obstore>=0.11.0", ] diff --git a/extralit-server/src/extralit_server/contexts/ocr/arrow.py b/extralit-server/src/extralit_server/contexts/ocr/arrow.py index 60136d74a..38fd5eba6 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/arrow.py +++ b/extralit-server/src/extralit_server/contexts/ocr/arrow.py @@ -10,6 +10,7 @@ from typing import Any, Optional import pyarrow as pa +from docling_core.transforms.serializer.markdown import MarkdownDocSerializer, MarkdownParams from docling_core.types.doc import DoclingDocument from docling_core.types.doc.document import DocItem, TableItem @@ -29,6 +30,7 @@ ("charspan_start", pa.int32()), ("charspan_end", pa.int32()), ("text", pa.string()), + ("markdown", pa.string()), ("html", pa.string()), ] ) @@ -58,9 +60,20 @@ def _table_html(doc: DoclingDocument, item: DocItem) -> Optional[str]: return None +def _serialize(serializer: MarkdownDocSerializer, item: DocItem) -> Optional[str]: + try: + return serializer.serialize(item=item).text + except Exception: # NULL, not '': a row the renderer could not handle is not a blank one + return None + + def item_rows(doc: DoclingDocument, document_id: str) -> list[dict[str, Any]]: """Flatten every item's provenance into rows matching `ITEM_SCHEMA`.""" rows: list[dict[str, Any]] = [] + # Raw characters and no image placeholder: this column feeds a search index, not a reader. + markdown = MarkdownDocSerializer( + doc=doc, params=MarkdownParams(escape_html=False, escape_underscores=False, image_placeholder="") + ) for reading_order, (item, _level) in enumerate(doc.iterate_items(with_groups=False)): base = { @@ -72,6 +85,7 @@ def item_rows(doc: DoclingDocument, document_id: str) -> list[dict[str, Any]]: "level": getattr(item, "level", None), "reading_order": reading_order, "text": getattr(item, "text", None) or None, + "markdown": _serialize(markdown, item), "html": _table_html(doc, item), } diff --git a/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py b/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py index cf2bc03b0..451d3ac49 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py +++ b/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py @@ -20,7 +20,7 @@ Size, TableCell, ) -from docling_core.types.doc.document import DocumentOrigin, NodeItem +from docling_core.types.doc.document import DocumentOrigin, FloatingItem, NodeItem, PictureItem, TableItem PDF_MIMETYPE = "application/pdf" @@ -34,6 +34,9 @@ #: Captions and footnotes legitimately overlap their figure, so containment must not drop them. CONTAINMENT_EXEMPT_LABELS = frozenset({DocItemLabel.CAPTION, DocItemLabel.FOOTNOTE}) +#: How far either side of a caption its figure or table may sit, nearest first, before after. +CAPTION_REACH = (-1, 1, -2, 2) + @dataclass(frozen=True) class LayoutBlock: @@ -153,6 +156,29 @@ def sort_body_by_position(doc: DoclingDocument) -> None: doc.body.children.sort(key=lambda ref: _sort_key(doc, ref.resolve(doc))) +def _page_of(item: NodeItem) -> Optional[int]: + provs = getattr(item, "prov", None) or [] + return provs[0].page_no if provs else None + + +def link_captions(doc: DoclingDocument) -> None: + """Attach each unlinked caption to the nearest table or picture on its page, looking back first. + + docling serializes a linked caption inside its owner and nowhere else, so this is what keeps a + caption out of the prose stream and inside the table or figure that it names. + """ + body = [ref.resolve(doc) for ref in doc.body.children] + linked = {ref.cref for item in body if isinstance(item, FloatingItem) for ref in item.captions} + for index, item in enumerate(body): + if getattr(item, "label", None) != DocItemLabel.CAPTION or item.self_ref in linked: + continue + for offset in CAPTION_REACH: + owner = body[index + offset] if 0 <= index + offset < len(body) else None + if isinstance(owner, (TableItem, PictureItem)) and _page_of(owner) == _page_of(item): + owner.captions.append(item.get_ref()) + break + + def append_blocks( doc: DoclingDocument, ctx: PageContext, @@ -185,4 +211,5 @@ def append_blocks( added.append(item) sort_body_by_position(doc) + link_captions(doc) return added diff --git a/extralit-server/src/extralit_server/contexts/ocr/elements.py b/extralit-server/src/extralit_server/contexts/ocr/elements.py new file mode 100644 index 000000000..b62c5b79d --- /dev/null +++ b/extralit-server/src/extralit_server/contexts/ocr/elements.py @@ -0,0 +1,139 @@ +"""Typed view of the layout store's `items` rows, one step below chunking. + +`arrow.item_rows` flattens a `DoclingDocument` into columns; this reads those columns back as +the three units a chunker dispatches on — a markdown run, a table, or a figure with its caption +— so `contexts/retrieval` never has to know a docling label. Rows in, rows out: chunking re-runs +from the Lance dataset without re-parsing the PDF. + +Rendering is docling's, done once at projection time into the `markdown` column. What this adds +is the heading breadcrumb, as window functions over the `items` columns, so a whole workspace is +projected in the same columnar pass as a single document. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional + +import pyarrow as pa +from docling_core.types.doc import DocItemLabel + +from extralit_server.contexts.ocr.docling_builder import PICTURE_LABELS, TABLE_LABELS + +if TYPE_CHECKING: + import duckdb + +MARKDOWN = "markdown" +TABLE = "table" +FIGURE = "figure" + +#: Running page furniture: repeated on every page, retrievable on none of them. +SKIPPED_LABELS = frozenset({DocItemLabel.PAGE_HEADER, DocItemLabel.PAGE_FOOTER}) + +#: Headings open a breadcrumb slot. A title sits above every section header, whatever its level. +TITLE_SLOT = 0 + +#: The deepest breadcrumb slot; anything deeper is nested under the same ancestor anyway. +MAX_HEADING_LEVEL = 6 + +ELEMENT_SCHEMA = pa.schema( + [ + ("document_id", pa.string()), + ("type", pa.string()), + ("content", pa.string()), + ("page_no", pa.int32()), + ("bbox", pa.list_(pa.float32(), 4)), + ("label", pa.string()), + ("level", pa.int8()), + ("item_ref", pa.string()), + ("reading_order", pa.int32()), + ("headings", pa.list_(pa.string())), + ("heading_level", pa.int8()), + ] +) + + +def _labels(labels: frozenset[DocItemLabel]) -> str: + """A label set as a SQL `IN` list. Enum values only — nothing here is caller input.""" + return "(" + ", ".join(f"'{label.value}'" for label in sorted(labels)) + ")" + + +def elements_sql(source: str = "items") -> str: + """The projection, as one statement over anything DuckDB can scan with the `items` columns.""" + slots = range(TITLE_SLOT, MAX_HEADING_LEVEL + 1) + # Per slot: the latest heading opened there, and the depth of the latest heading at or above + # it. The slot is still in scope only while those coincide. + breadcrumb = ",\n ".join( + f"last_value(CASE WHEN slot = {s} THEN own_text END IGNORE NULLS) OVER running AS heading_{s},\n " + f"last_value(CASE WHEN slot <= {s} THEN slot END IGNORE NULLS) OVER running AS newest_{s}" + for s in slots + ) + headings = ", ".join(f"CASE WHEN newest_{s} = {s} THEN heading_{s} END" for s in slots) + # Python's `str.strip`; DuckDB's bare `trim` takes spaces off and leaves newlines behind. + strip = r"regexp_replace({}, '^\s+|\s+$', '', 'g')" + + return f""" +WITH sliced AS ( + SELECT * EXCLUDE (label), + coalesce(label, '{DocItemLabel.TEXT.value}') AS label, + row_number() OVER (PARTITION BY document_id ORDER BY reading_order, prov_index) AS ord, + -- A charspan short of the whole text is one page's share of an item that spans a break. + coalesce(charspan_start > 0 OR charspan_end < length(text), false) AS partial, + {strip.format("CASE WHEN partial THEN text[charspan_start + 1 : charspan_end] ELSE coalesce(text, '') END")} + AS own_text, + -- Markdown is rendered per item, so a partial row falls back to its share of the raw text. + CASE + WHEN markdown = '' THEN '' + WHEN partial OR markdown IS NULL THEN own_text + ELSE {strip.format("markdown")} + END AS content, + CASE + WHEN own_text = '' THEN NULL + WHEN label = '{DocItemLabel.TITLE.value}' THEN {TITLE_SLOT} + WHEN label = '{DocItemLabel.SECTION_HEADER.value}' + THEN least(greatest(coalesce(level, 1), 1), {MAX_HEADING_LEVEL}) + END AS slot + FROM {source} +), +breadcrumbs AS ( + SELECT *, + {breadcrumb} + FROM sliced + WINDOW running AS ( + PARTITION BY document_id ORDER BY ord ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ) +) +SELECT + document_id, + CASE + WHEN label IN {_labels(TABLE_LABELS)} THEN '{TABLE}' + WHEN label IN {_labels(PICTURE_LABELS)} THEN '{FIGURE}' + ELSE '{MARKDOWN}' + END AS type, + content, page_no, bbox, label, level, self_ref AS item_ref, reading_order, + list_filter([{headings}], heading -> heading IS NOT NULL) AS headings, + newest_{MAX_HEADING_LEVEL} AS heading_level +FROM breadcrumbs +WHERE label NOT IN {_labels(SKIPPED_LABELS)} AND content <> '' +ORDER BY document_id, ord +""" + + +def elements_table(items: Any, *, connection: Optional[duckdb.DuckDBPyConnection] = None) -> pa.Table: + """Project `items` rows into elements, in reading order, one per provenance row. + + An item spanning a page break stays two elements with two bboxes rather than one element + claiming to be in two places. `items` is anything DuckDB registers — an Arrow table, a Lance + dataset — and every document in it is projected in the same pass. + """ + import duckdb + + own = connection is None + connection = connection or duckdb.connect() + try: + connection.register("_items", items) + table = connection.execute(elements_sql("_items")).to_arrow_table() + finally: + connection.unregister("_items") + if own: + connection.close() + return table.cast(ELEMENT_SCHEMA) diff --git a/extralit-server/src/extralit_server/contexts/ocr/layout_store.py b/extralit-server/src/extralit_server/contexts/ocr/layout_store.py index c2b2f96e2..6396035b9 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/layout_store.py +++ b/extralit-server/src/extralit_server/contexts/ocr/layout_store.py @@ -44,6 +44,18 @@ _SCHEMAS = {ITEMS_DATASET: ITEM_SCHEMA, PAGES_DATASET: PAGE_SCHEMA} +# Lance's SQL type names for `add_columns`; anything else needs a real migration, not a NULL fill. +_SQL_TYPES = { + pa.string(): "string", + pa.bool_(): "boolean", + pa.int8(): "tinyint", + pa.int16(): "smallint", + pa.int32(): "int", + pa.int64(): "bigint", + pa.float32(): "float", + pa.float64(): "double", +} + def layout_root(workspace_name: str) -> tuple[str, Optional[dict[str, str]]]: """Root of the workspace's datasets, resolved exactly like every other artifact of it.""" @@ -138,6 +150,14 @@ def open(self, name: str) -> Optional[lance.LanceDataset]: def _write(self, name: str, data: pa.Table, mode: str) -> int: return lance.write_dataset(data, self.uri(name), mode=mode, storage_options=self.storage_options).version + def _add_missing_columns(self, dataset: lance.LanceDataset, schema: pa.Schema) -> lance.LanceDataset: + """Widen a dataset written under an older schema; new columns are NULL for the rows already there.""" + missing = [field for field in schema if field.name not in dataset.schema.names] + if not missing: + return dataset + dataset.add_columns({field.name: f"CAST(NULL AS {_SQL_TYPES[field.type]})" for field in missing}) + return lance.dataset(dataset.uri, storage_options=self.storage_options) + def _replace_one(self, name: str, document_id: UUID | str, data: pa.Table) -> int: dataset = self.open(name) if dataset is None: @@ -147,6 +167,7 @@ def _replace_one(self, name: str, document_id: UUID | str, data: pa.Table) -> in # Another worker created it between the open and the write; join it instead. dataset = lance.dataset(self.uri(name), storage_options=self.storage_options) + dataset = self._add_missing_columns(dataset, data.schema) dataset.delete(_document_filter(document_id)) if data.num_rows == 0: return lance.dataset(self.uri(name), storage_options=self.storage_options).version diff --git a/extralit-server/tests/integration/conftest.py b/extralit-server/tests/integration/conftest.py index affdadb64..96fb82002 100644 --- a/extralit-server/tests/integration/conftest.py +++ b/extralit-server/tests/integration/conftest.py @@ -10,7 +10,7 @@ import pytest import pytest_asyncio -from httpx import AsyncClient +from httpx import ASGITransport, AsyncClient from extralit_server.constants import API_KEY_HEADER_NAME from extralit_server.database import get_async_db @@ -39,7 +39,7 @@ async def override_get_async_db(): api_v1.dependency_overrides[get_async_db] = override_get_async_db - async with AsyncClient(app=app, base_url="http://testserver") as client: + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://testserver") as client: yield client # Pop only what this fixture registered. `tests/unit/conftest.py`'s async_client writes diff --git a/extralit-server/tests/unit/api/handlers/v1/test_datasets.py b/extralit-server/tests/unit/api/handlers/v1/test_datasets.py index 1b0603135..02d23a6aa 100644 --- a/extralit-server/tests/unit/api/handlers/v1/test_datasets.py +++ b/extralit-server/tests/unit/api/handlers/v1/test_datasets.py @@ -1,3 +1,4 @@ +import json import math import uuid from datetime import datetime @@ -82,6 +83,11 @@ from sqlalchemy.ext.asyncio import AsyncSession +def nan_body(payload: dict, headers: dict) -> dict: + """Post a body the JSON spec forbids. `json=` cannot: httpx encodes with allow_nan=False.""" + return {"content": json.dumps(payload), "headers": {**headers, "Content-Type": "application/json"}} + + @pytest.mark.asyncio class TestSuiteDatasets: async def test_list_current_user_datasets(self, async_client: "AsyncClient", owner_auth_header: dict) -> None: @@ -1959,7 +1965,7 @@ async def test_create_dataset_records_with_metadata_nan_values( } response = await async_client.post( - f"/api/v1/datasets/{dataset.id}/records/bulk", headers=owner_auth_header, json=records_json + f"/api/v1/datasets/{dataset.id}/records/bulk", **nan_body(records_json, owner_auth_header) ) assert response.status_code == 422 @@ -3027,23 +3033,25 @@ async def test_update_dataset_records_with_metadata_nan_value( response = await async_client.put( f"/api/v1/datasets/{dataset.id}/records/bulk", - headers=owner_auth_header, - json={ - "items": [ - { - "id": str(records[0].id), - "metadata": {"terms": math.nan}, - }, - { - "id": str(records[1].id), - "metadata": {"float": math.nan}, - }, - { - "id": str(records[2].id), - "metadata": {"terms": "a"}, - }, - ] - }, + **nan_body( + { + "items": [ + { + "id": str(records[0].id), + "metadata": {"terms": math.nan}, + }, + { + "id": str(records[1].id), + "metadata": {"float": math.nan}, + }, + { + "id": str(records[2].id), + "metadata": {"terms": "a"}, + }, + ] + }, + owner_auth_header, + ), ) assert response.status_code == 422 diff --git a/extralit-server/tests/unit/conftest.py b/extralit-server/tests/unit/conftest.py index 8d7c934c3..5bc473561 100644 --- a/extralit-server/tests/unit/conftest.py +++ b/extralit-server/tests/unit/conftest.py @@ -3,7 +3,7 @@ import pytest import pytest_asyncio -from httpx import AsyncClient +from httpx import ASGITransport, AsyncClient from opensearchpy import OpenSearch from sqlalchemy.engine.interfaces import IsolationLevel @@ -95,7 +95,7 @@ async def override_get_search_engine(): } ) - async with AsyncClient(app=app, base_url="http://testserver") as async_client: + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://testserver") as async_client: yield async_client # Clear from `api_v1` — that is where the overrides above were registered. Clearing diff --git a/extralit-server/tests/unit/contexts/ocr/test_arrow.py b/extralit-server/tests/unit/contexts/ocr/test_arrow.py index 71e8c431e..286e7d46c 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_arrow.py +++ b/extralit-server/tests/unit/contexts/ocr/test_arrow.py @@ -145,6 +145,35 @@ def test_text_items_have_no_html(self, doc): assert rows["#/texts/2"]["html"] is None + def test_markdown_is_docling_rendering_of_each_item(self, doc): + rows = {r["self_ref"]: r for r in items_table(doc, DOCUMENT_ID).to_pylist()} + + assert rows["#/texts/0"]["markdown"] == "# A Paper" + assert rows["#/texts/1"]["markdown"] == "### Methods" + assert rows["#/texts/2"]["markdown"] == "Body text here." + assert rows["#/tables/0"]["markdown"].startswith("| Group") + assert rows["#/pictures/0"]["markdown"] == "" + + def test_markdown_keeps_the_characters_a_search_index_needs(self): + document = new_document("sample") + ctx = PageContext(page_no=1, size=Size(width=612, height=792)) + append_blocks( + document, ctx, [LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text="p_value < 0.05 & n_obs")] + ) + + rows = items_table(document, DOCUMENT_ID).to_pylist() + + assert rows[0]["markdown"] == "p_value < 0.05 & n_obs" + + def test_markdown_is_rendered_once_per_item_not_once_per_provenance(self, doc): + for item in doc.texts: + if item.self_ref == "#/texts/2": + item.prov.append(ProvenanceItem(page_no=2, bbox=bbox(t=10, b=30), charspan=(0, 0))) + + rows = [r for r in items_table(doc, DOCUMENT_ID).to_pylist() if r["self_ref"] == "#/texts/2"] + + assert [r["markdown"] for r in rows] == ["Body text here."] * 2 + def test_parent_ref_is_recorded(self, doc): rows = {r["self_ref"]: r for r in items_table(doc, DOCUMENT_ID).to_pylist()} diff --git a/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py b/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py index 2de355d5c..22e9e1721 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py +++ b/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py @@ -213,6 +213,80 @@ def test_captions_are_kept_even_when_they_touch_a_figure(self, doc, ctx): assert [t.text for t in doc.texts] == ["Figure 1."] + def test_a_caption_is_linked_to_the_figure_it_touches(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=100, b=300, left=0, right=500)), + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=310, b=320), text="Figure 1."), + ] + + append_blocks(doc, ctx, blocks) + + assert doc.pictures[0].caption_text(doc) == "Figure 1." + + def test_a_caption_above_its_table_binds_to_it(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=90, b=99), text="Table 1."), + LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=300)), + ] + + append_blocks(doc, ctx, blocks) + + assert doc.tables[0].caption_text(doc) == "Table 1." + + def test_the_nearer_neighbour_wins_when_a_caption_sits_between_two_figures(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=100, b=200)), + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=210, b=220), text="Belongs to the first."), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=230, b=240), text="Prose."), + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=300, b=400)), + ] + + append_blocks(doc, ctx, blocks) + + assert doc.pictures[0].caption_text(doc) == "Belongs to the first." + assert doc.pictures[1].captions == [] + + def test_a_caption_reaches_past_one_line_of_prose(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=100, b=200)), + LayoutBlock(label=DocItemLabel.FOOTNOTE, bbox=bbox(t=205, b=208), text="a"), + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=210, b=220), text="Figure 1."), + ] + + append_blocks(doc, ctx, blocks) + + assert doc.pictures[0].caption_text(doc) == "Figure 1." + + def test_two_captions_on_one_figure_are_both_linked(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=90, b=99), text="Figure 1."), + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=100, b=200)), + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=210, b=220), text="A red square."), + ] + + append_blocks(doc, ctx, blocks) + + assert [ref.resolve(doc).text for ref in doc.pictures[0].captions] == ["Figure 1.", "A red square."] + + def test_a_caption_never_binds_across_a_page(self, doc, ctx): + append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=700, b=780))]) + page_two = PageContext(page_no=2, size=ctx.size) + + append_blocks(doc, page_two, [LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=10, b=20), text="Orphan.")]) + + assert doc.pictures[0].captions == [] + + def test_a_second_pass_over_a_page_links_nothing_twice(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=100, b=200)), + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=210, b=220), text="Figure 1."), + ] + append_blocks(doc, ctx, blocks) + + append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=400, b=420), text="More.")]) + + assert len(doc.pictures[0].captions) == 1 + def test_every_item_carries_a_full_provenance_triple(self, doc, ctx): blocks = [ LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text="hello"), diff --git a/extralit-server/tests/unit/contexts/ocr/test_elements.py b/extralit-server/tests/unit/contexts/ocr/test_elements.py new file mode 100644 index 000000000..90734459c --- /dev/null +++ b/extralit-server/tests/unit/contexts/ocr/test_elements.py @@ -0,0 +1,330 @@ +"""Tests for the columnar element view of `items` rows.""" + +import pyarrow as pa +import pytest +from docling_core.types.doc import BoundingBox, CoordOrigin, DocItemLabel, Size + +from extralit_server.contexts.ocr.arrow import ITEM_SCHEMA, item_rows +from extralit_server.contexts.ocr.docling_builder import ( + LayoutBlock, + PageContext, + append_blocks, + new_document, +) +from extralit_server.contexts.ocr.elements import ( + ELEMENT_SCHEMA, + FIGURE, + MARKDOWN, + TABLE, + elements_table, +) +from extralit_server.contexts.ocr.tables import make_cell + +DOCUMENT_ID = "11111111-2222-3333-4444-555555555555" + + +def bbox(t: float, b: float, left: float = 10.0, right: float = 100.0) -> BoundingBox: + return BoundingBox(l=left, t=t, r=right, b=b, coord_origin=CoordOrigin.TOPLEFT) + + +def row(label, reading_order, text=None, markdown=None, **overrides): + """An `items` row. `markdown` defaults to `text`, the way docling renders plain prose.""" + base = { + "document_id": DOCUMENT_ID, + "self_ref": f"#/texts/{reading_order}", + "parent_ref": None, + "label": label, + "content_layer": "body", + "level": None, + "reading_order": reading_order, + "prov_index": 0, + "page_no": 1, + "bbox": [0.0, 0.0, 10.0, 10.0], + "coord_origin": "TOPLEFT", + "text": text, + "markdown": text if markdown is None else markdown, + "html": None, + "charspan_start": None, + "charspan_end": None, + } + base.update(overrides) + return base + + +def elements(rows) -> list[dict]: + """The projection as dicts — assertions read better than Arrow column slices.""" + return elements_table(pa.Table.from_pylist(list(rows), schema=ITEM_SCHEMA)).to_pylist() + + +class TestSchema: + def test_the_projection_conforms_to_the_declared_schema(self): + table = elements_table(pa.Table.from_pylist([row(DocItemLabel.TEXT, 0, text="x")], schema=ITEM_SCHEMA)) + + assert table.schema == ELEMENT_SCHEMA + + def test_an_empty_input_yields_an_empty_table_not_an_error(self): + table = elements_table(pa.Table.from_pylist([], schema=ITEM_SCHEMA)) + + assert table.num_rows == 0 + assert table.schema == ELEMENT_SCHEMA + + +class TestHeadingBreadcrumb: + def test_breadcrumb_deepens_and_unwinds_with_heading_level(self): + rows = [ + row(DocItemLabel.TITLE, 0, text="A Paper"), + row(DocItemLabel.SECTION_HEADER, 1, level=1, text="Methods"), + row(DocItemLabel.SECTION_HEADER, 2, level=2, text="Sampling"), + row(DocItemLabel.TEXT, 3, text="Deep body."), + row(DocItemLabel.SECTION_HEADER, 4, level=1, text="Results"), + row(DocItemLabel.TEXT, 5, text="Shallow body."), + ] + + assert [e["headings"] for e in elements(rows)] == [ + ["A Paper"], + ["A Paper", "Methods"], + ["A Paper", "Methods", "Sampling"], + ["A Paper", "Methods", "Sampling"], + # Results closes Methods and Sampling but never the title. + ["A Paper", "Results"], + ["A Paper", "Results"], + ] + + def test_a_heading_carries_itself_so_a_chunk_knows_its_own_path(self): + rows = [row(DocItemLabel.SECTION_HEADER, 0, level=1, text="Methods", markdown="## Methods")] + + assert elements(rows)[0]["headings"] == ["Methods"] + + def test_the_breadcrumb_is_the_raw_heading_text_not_its_markdown(self): + rows = [ + row(DocItemLabel.SECTION_HEADER, 0, level=1, text="Methods", markdown="## Methods"), + row(DocItemLabel.TEXT, 1, text="Body."), + ] + + assert [e["headings"] for e in elements(rows)] == [["Methods"], ["Methods"]] + + def test_heading_level_tracks_the_deepest_slot_in_scope(self): + rows = [ + row(DocItemLabel.TITLE, 0, text="A Paper"), + row(DocItemLabel.SECTION_HEADER, 1, level=2, text="Sampling"), + row(DocItemLabel.TEXT, 2, text="Body."), + ] + + assert [e["heading_level"] for e in elements(rows)] == [0, 2, 2] + + def test_prose_before_any_heading_has_no_breadcrumb(self): + element = elements([row(DocItemLabel.TEXT, 0, text="Orphan.")])[0] + assert element["headings"] == [] + assert element["heading_level"] is None + + def test_a_blank_heading_opens_no_slot(self): + rows = [ + row(DocItemLabel.SECTION_HEADER, 0, level=1, text=" ", markdown=""), + row(DocItemLabel.TEXT, 1, text="Body."), + ] + + assert [e["headings"] for e in elements(rows)] == [[]] + + def test_levels_past_h6_share_the_deepest_slot(self): + rows = [ + row(DocItemLabel.SECTION_HEADER, 0, level=7, text="Seven"), + row(DocItemLabel.SECTION_HEADER, 1, level=99, text="Ninety-nine"), + ] + + assert [e["headings"] for e in elements(rows)] == [["Seven"], ["Ninety-nine"]] + + +class TestContent: + def test_content_is_the_rendered_markdown_not_the_raw_text(self): + rows = [row(DocItemLabel.SECTION_HEADER, 0, level=1, text="Methods", markdown="## Methods")] + + element = elements(rows)[0] + + assert element["type"] == MARKDOWN + assert element["content"] == "## Methods" + + def test_surrounding_whitespace_is_stripped_including_newlines(self): + assert elements([row(DocItemLabel.TEXT, 0, text=" \n\tProse.\n ")])[0]["content"] == "Prose." + + def test_a_row_rendered_blank_is_dropped(self): + # A linked caption renders as '' on its own: docling folds it into the table or figure. + assert elements([row(DocItemLabel.CAPTION, 0, text="Figure 1.", markdown="")]) == [] + + def test_a_row_never_rendered_falls_back_to_its_text(self): + # NULL is a row written before the column existed, or one the renderer could not handle. + assert elements([row(DocItemLabel.TEXT, 0, text="Old prose.", markdown="")]) == [] + rows = [{**row(DocItemLabel.TEXT, 0, text="Old prose."), "markdown": None}] + + assert [e["content"] for e in elements(rows)] == ["Old prose."] + + def test_tables_and_figures_carry_their_rendering_whatever_their_text(self): + rows = [ + row(DocItemLabel.TABLE, 0, markdown="Table 1.\n\n| N |\n|---|\n| 1 |"), + row(DocItemLabel.PICTURE, 1, markdown="Figure 1."), + row(DocItemLabel.PICTURE, 2, markdown=""), + ] + + assert [(e["type"], e["content"]) for e in elements(rows)] == [ + (TABLE, "Table 1.\n\n| N |\n|---|\n| 1 |"), + (FIGURE, "Figure 1."), + ] + + +class TestProvenance: + def test_running_headers_and_footers_are_dropped(self): + rows = [ + row(DocItemLabel.PAGE_HEADER, 0, text="Journal of Things"), + row(DocItemLabel.TEXT, 1, text="Real body."), + row(DocItemLabel.PAGE_FOOTER, 2, text="7"), + ] + + assert [e["content"] for e in elements(rows)] == ["Real body."] + + def test_one_element_per_provenance_row_when_an_item_spans_a_page_break(self): + text = "first half second half" + spanning = [ + row(DocItemLabel.TEXT, 0, page_no=1, text=text, charspan_start=0, charspan_end=11), + row(DocItemLabel.TEXT, 0, page_no=2, prov_index=1, text=text, charspan_start=11, charspan_end=22), + ] + + assert [(e["page_no"], e["content"]) for e in elements(spanning)] == [(1, "first half"), (2, "second half")] + + def test_a_page_spanning_heading_keeps_its_slice_and_loses_its_marker(self): + # Markdown is rendered per item and cannot be sliced by charspan, so the raw slice wins. + text = "Methods and Materials" + spanning = [ + row( + DocItemLabel.SECTION_HEADER, + 0, + level=1, + text=text, + markdown=f"## {text}", + charspan_start=0, + charspan_end=7, + ), + row( + DocItemLabel.SECTION_HEADER, + 0, + level=1, + page_no=2, + prov_index=1, + text=text, + markdown=f"## {text}", + charspan_start=7, + charspan_end=21, + ), + ] + + assert [e["content"] for e in elements(spanning)] == ["Methods", "and Materials"] + + def test_a_page_spanning_table_repeats_its_rendering_on_both_pages(self): + spanning = [ + row(DocItemLabel.TABLE, 0, page_no=1, markdown="| N |", charspan_start=0, charspan_end=0), + row(DocItemLabel.TABLE, 0, page_no=2, prov_index=1, markdown="| N |", charspan_start=0, charspan_end=0), + ] + + assert [(e["page_no"], e["content"]) for e in elements(spanning)] == [(1, "| N |"), (2, "| N |")] + + def test_elements_come_back_in_reading_order_whatever_the_row_order(self): + rows = [row(DocItemLabel.TEXT, 2, text="third"), row(DocItemLabel.TEXT, 0, text="first")] + + assert [e["content"] for e in elements(rows)] == ["first", "third"] + + def test_bbox_and_item_ref_survive_the_round_trip(self): + element = elements([row(DocItemLabel.TEXT, 0, text="x", bbox=[1.0, 2.0, 3.0, 4.0])])[0] + + assert element["bbox"] == [1.0, 2.0, 3.0, 4.0] + assert element["item_ref"] == "#/texts/0" + + +class TestManyDocuments: + def test_documents_are_projected_in_one_pass_without_bleeding_into_each_other(self): + rows = [] + for document_id in ("aaaa", "bbbb"): + for base in ( + row(DocItemLabel.SECTION_HEADER, 0, level=1, text=f"{document_id} heading"), + row(DocItemLabel.TEXT, 1, text=f"{document_id} body"), + ): + rows.append({**base, "document_id": document_id}) + + found = elements(reversed(rows)) + + assert [(e["document_id"], e["headings"]) for e in found] == [ + ("aaaa", ["aaaa heading"]), + ("aaaa", ["aaaa heading"]), + ("bbbb", ["bbbb heading"]), + ("bbbb", ["bbbb heading"]), + ] + + +class TestAgainstRealProjection: + @pytest.fixture + def found(self): + document = new_document("sample") + append_blocks( + document, + PageContext(page_no=1, size=Size(width=612, height=792)), + [ + LayoutBlock(label=DocItemLabel.TITLE, bbox=bbox(t=1, b=5), text="A Paper"), + LayoutBlock(label=DocItemLabel.SECTION_HEADER, bbox=bbox(t=10, b=40), text="Methods", level=1), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=50, b=70), text="Body text."), + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=190, b=199), text="Table 1. Counts."), + LayoutBlock( + label=DocItemLabel.TABLE, + bbox=bbox(t=200, b=400), + cells=[make_cell("N", row=0, col=0, column_header=True), make_cell("42", row=1, col=0)], + ), + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=500, b=600)), + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=601, b=610), text="Figure 1. A red square."), + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=700, b=750)), + ], + ) + return elements(item_rows(document, DOCUMENT_ID)) + + def test_captions_ride_inside_their_table_and_figure_not_the_prose(self, found): + assert [e["type"] for e in found] == [MARKDOWN, MARKDOWN, MARKDOWN, TABLE, FIGURE] + assert found[3]["content"].startswith("Table 1. Counts.\n\n|") + assert "|-----|" in found[3]["content"] and "42" in found[3]["content"] + assert found[4]["content"] == "Figure 1. A red square." + + def test_an_uncaptioned_figure_yields_nothing_retrievable(self, found): + assert sum(e["type"] == FIGURE for e in found) == 1 + + def test_docling_reserves_the_top_heading_for_the_title(self, found): + assert [e["content"] for e in found[:2]] == ["# A Paper", "## Methods"] + + def test_the_breadcrumb_reaches_every_element(self, found): + assert all(e["headings"] == ["A Paper", "Methods"] for e in found[1:]) + + +class TestTableChunking: + """chonkie owns the row-window split; pin the behaviour the element design leans on.""" + + def test_a_table_split_into_row_windows_repeats_caption_and_header_on_every_chunk(self): + from chonkie import TableChunker + + document = new_document("sample") + append_blocks( + document, + PageContext(page_no=1, size=Size(width=612, height=792)), + [ + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=190, b=199), text="Table 1. Counts."), + LayoutBlock( + label=DocItemLabel.TABLE, + bbox=bbox(t=200, b=400), + cells=[ + make_cell("Group", row=0, col=0, column_header=True), + make_cell("N", row=0, col=1, column_header=True), + *[make_cell(f"g{i}", row=i, col=0) for i in range(1, 7)], + *[make_cell(str(i), row=i, col=1) for i in range(1, 7)], + ], + ), + ], + ) + content = elements(item_rows(document, DOCUMENT_ID))[0]["content"] + + chunks = [chunk.text for chunk in TableChunker(chunk_size=2)(content)] + + assert len(chunks) > 1 + assert all("Table 1. Counts." in chunk and "| Group" in chunk for chunk in chunks) + assert "| g6" in chunks[-1] and "| g6" not in chunks[0] diff --git a/extralit-server/tests/unit/contexts/ocr/test_layout_store.py b/extralit-server/tests/unit/contexts/ocr/test_layout_store.py index 840870c86..846c99615 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_layout_store.py +++ b/extralit-server/tests/unit/contexts/ocr/test_layout_store.py @@ -6,6 +6,7 @@ import time from uuid import uuid4 +import lance import pyarrow as pa import pytest @@ -45,6 +46,7 @@ def items(document_id: str, count: int = 3, label: str = "text") -> pa.Table: "charspan_start": 0, "charspan_end": 5, "text": f"row {i}", + "markdown": f"row {i}", "html": None, } for i in range(count) @@ -129,6 +131,17 @@ def test_a_zero_row_document_still_clears_its_old_rows(self, local_store): assert local_store.load_items(document_id).num_rows == 0 assert local_store.load_pages(document_id).num_rows == 0 + def test_a_dataset_written_under_an_older_schema_is_widened_not_rejected(self, local_store): + old, new = str(uuid4()), str(uuid4()) + narrow = items(old, 2).drop_columns(["markdown"]) + lance.write_dataset(narrow, local_store.items_uri(), mode="create") + + local_store.replace_document(new, items(new, 1), pages(new)) + + assert set(local_store.open(ITEMS_DATASET).schema.names) == set(ITEM_SCHEMA.names) + assert local_store.load_items(old, columns=["markdown"]).to_pylist() == [{"markdown": None}] * 2 + assert local_store.load_items(new, columns=["markdown"]).to_pylist() == [{"markdown": "row 0"}] + def test_replace_reports_the_dataset_versions(self, local_store): document_id = str(uuid4()) diff --git a/extralit-server/uv.lock b/extralit-server/uv.lock index c33ac8487..16c719caa 100644 --- a/extralit-server/uv.lock +++ b/extralit-server/uv.lock @@ -608,6 +608,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/68/687187c7e26cb24ccbd88e5069f5ef00eba804d36dde11d99aad0838ab45/charset_normalizer-3.4.6-py3-none-any.whl", hash = "sha256:947cf925bc916d90adba35a64c82aace04fa39b46b52d4630ece166655905a69", size = 61455, upload-time = "2026-03-15T18:53:23.833Z" }, ] +[[package]] +name = "chonkie" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "chonkie-core" }, + { name = "httpx" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "tenacity" }, + { name = "tokie" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/67/e6ceace2b2066cc38206c81e9105735975d7a58d89c9e7a871fe9acef63d/chonkie-1.7.0.tar.gz", hash = "sha256:4352aa214b66376c32523e524b94b7f1456a5e7611c48037dadb45e46f436b18", size = 190834, upload-time = "2026-07-07T10:06:02.475Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/c0/b5f91fb340d354a8c20e2f65d08b9dab357360da78d230e96c6d79b36a2e/chonkie-1.7.0-py3-none-any.whl", hash = "sha256:98822ca80fbf3c0775f59329a4f6b6bfedb34edff004b5e04e53c3281369b921", size = 233764, upload-time = "2026-07-07T10:06:00.824Z" }, +] + +[[package]] +name = "chonkie-core" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/17/ad31bbcfe1a7b63f76e065684a8e9cd98552d3c627fb28d8fbc05f1a9456/chonkie_core-0.10.2.tar.gz", hash = "sha256:c8e40ef8f3a034a7c5dd23a0401dce2ef2b4883f5a6a29cf94176d64b209bdbb", size = 69966, upload-time = "2026-05-28T19:16:12.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/a0/16ff6f6d3ddfb3f39d89663717e91eaa1bd5ba6ac52ec2fa01795ac33c65/chonkie_core-0.10.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b63d25a3270a98eb5a6739072315f36566c87139078797a0970bc8d480f5bbe0", size = 349233, upload-time = "2026-05-28T19:15:38.805Z" }, + { url = "https://files.pythonhosted.org/packages/ed/9d/2ed7af6666a168fcb18328e06354035f506aa8965a479827aa6f5528b189/chonkie_core-0.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:86c3e04603ce69d380f3f6a8f14f85dd902410ad1ec86709ddcfad43dc968946", size = 339167, upload-time = "2026-05-28T19:15:40.388Z" }, + { url = "https://files.pythonhosted.org/packages/12/bd/373ca97e84a4c4caf9adba9e710bc48f50c87524379a949cd17ecc588def/chonkie_core-0.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2225d6e6622b26d4542ed9661e71b51dfd8b4c5ae7e652fe85b4f0faa4be8eba", size = 390116, upload-time = "2026-05-28T19:15:41.556Z" }, + { url = "https://files.pythonhosted.org/packages/09/62/c3ae371065d31e52eb96d05ef02384521c99dd6bd036bc4b1d7a09ea4cbb/chonkie_core-0.10.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:e7e05701463e8e7a339a7be617fe2306153dcb5f269f43251e1516f40ce91134", size = 375235, upload-time = "2026-05-28T19:15:42.763Z" }, + { url = "https://files.pythonhosted.org/packages/16/08/3f55dbb5cd033b9c33e9e5fbaa5ee88af3dbc78e6858cf42341c903c5cc6/chonkie_core-0.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:c7c2c431d86fbae7c1f8d0b8628236df27b4dab040dc6504ded209be9e36f5b0", size = 230868, upload-time = "2026-05-28T19:15:44.207Z" }, + { url = "https://files.pythonhosted.org/packages/c9/0b/4c28270b24d9e756c90a52966f26ddff5fa2f639aed7e6cd9cc1e1728176/chonkie_core-0.10.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a431af90ac4e4072e02699ac1300efcaaff2ee432b052fe51027998859af1055", size = 349504, upload-time = "2026-05-28T19:15:45.801Z" }, + { url = "https://files.pythonhosted.org/packages/93/a6/069b46520a23264585ddacd117bee62594afff0f851cd1dfedf32b900faf/chonkie_core-0.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:712ecc448302bdc8f3ab4aab1e17d5d37b50d57d89d4fa4ec44821a37eb8fa57", size = 339673, upload-time = "2026-05-28T19:15:46.992Z" }, + { url = "https://files.pythonhosted.org/packages/a5/05/513baf0c159fb93e6c05bf199cca79045b2c186e9fea1fd7ebf47d063c7b/chonkie_core-0.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff79cbe5016f85a8fac2164e1716f3335b631704bfa486f0d1de02c6d88c2bce", size = 390030, upload-time = "2026-05-28T19:15:48.434Z" }, + { url = "https://files.pythonhosted.org/packages/50/65/4a86e1648147df94bcc2d681b6c5f40a21c57df65b45bf2f3ceb72fcb784/chonkie_core-0.10.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:f69969fa3f75930bd10f1ec502993b758df4e5a62d170c373a4fa73743dc2f93", size = 374622, upload-time = "2026-05-28T19:15:49.836Z" }, + { url = "https://files.pythonhosted.org/packages/54/62/425bca737db62a371b1e3393d76f69f483baf3482ccd88504089501ee0e4/chonkie_core-0.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:cbe5a8a1e89a79a74bac99409e1bf5e7a5b3e2286ac5f2185077d39f5fa95175", size = 231028, upload-time = "2026-05-28T19:15:50.956Z" }, + { url = "https://files.pythonhosted.org/packages/56/2f/4dd88b4af9ef0e9ed31a3c6a3d8e44327c5ca22b074076830b0224e0ad58/chonkie_core-0.10.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:31f8fa3415d2e93cc3bdb1e99c3b6865278169af348c68fb2943aed2fcabc010", size = 347520, upload-time = "2026-05-28T19:15:52.331Z" }, + { url = "https://files.pythonhosted.org/packages/b2/3f/ba4083f21b4ef52ed130a79f371fed453b7edf998bf08079694137880bff/chonkie_core-0.10.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a6d081363fbfaad0f36c66e67e6dec1c9f9850c04c0475b64b3e47084257b646", size = 336897, upload-time = "2026-05-28T19:15:53.787Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9f/cbfec0f25e35d2a5b62a9cba7a5a85cfc5349210b9f9acb40832060211e4/chonkie_core-0.10.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51dbf2163dac5ad695dde3f92155c6febb9332b5fb5a174ac80ae9fc8f8ccde9", size = 387231, upload-time = "2026-05-28T19:15:54.851Z" }, + { url = "https://files.pythonhosted.org/packages/89/c8/b0d91419730bff4566305f2c85f27fb1378e8e1e99f8e59403c6b25d43a2/chonkie_core-0.10.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5a09bcb56d9d9f2fa6ec84485aeecbcaa375567eea9fbec18517d2ef643b9029", size = 371506, upload-time = "2026-05-28T19:15:56.172Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c2/05c2c85b11d092f04768c41fb9cfdd27a9d27fa7e10e107188a51ce8813f/chonkie_core-0.10.2-cp312-cp312-win_amd64.whl", hash = "sha256:ac686d192c4dd7e038cea4aba59c677d45e2b1e8de8eb1dd45d269b22a4ae201", size = 229686, upload-time = "2026-05-28T19:15:57.759Z" }, + { url = "https://files.pythonhosted.org/packages/59/4f/0814d2bd9abddbbc37f8c204f056d28f2ab7d14bf742515e9684062bd758/chonkie_core-0.10.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:25372879e43b235dee01f48309634a3fce07b55c382a7fbe200d6b66769267e4", size = 346848, upload-time = "2026-05-28T19:15:58.979Z" }, + { url = "https://files.pythonhosted.org/packages/3f/33/b640f406b6d2fe30d5c41d4a91411d9ed1dc22acf6497cda837a6f121ba4/chonkie_core-0.10.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3da60e797b2fcbef3e17fad3e46bb7d4f5541ac5f87a5556c2c610cef83a549a", size = 336613, upload-time = "2026-05-28T19:16:00.125Z" }, + { url = "https://files.pythonhosted.org/packages/e9/76/079296e83d2ed6e34b8ed24bd23e59193b502b53ddfa176b3e921410a5db/chonkie_core-0.10.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aebeba2d2cb0d30eefdbdb18d2ab2fcfbc0ea0d5121f61f537a6114a1da56101", size = 386412, upload-time = "2026-05-28T19:16:01.425Z" }, + { url = "https://files.pythonhosted.org/packages/04/6e/2c505276878b695b4cee07eb35a5c536ed92d60c722b4547da1517fb3e41/chonkie_core-0.10.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:72974163eab058d14c365f45b53d2367b1e04a0dcd779dedd828973cf21256d7", size = 371726, upload-time = "2026-05-28T19:16:02.8Z" }, + { url = "https://files.pythonhosted.org/packages/38/39/158f85b728d84e85a00768ccebe3a289d21462f52ba7850db9e01e61e4df/chonkie_core-0.10.2-cp313-cp313-win_amd64.whl", hash = "sha256:9f77811bb722bd019a52353364bc6dcea1a9998120a413f1c2a1c79153a66386", size = 229447, upload-time = "2026-05-28T19:16:04.21Z" }, + { url = "https://files.pythonhosted.org/packages/0b/45/e4d68840847133afa9c69fc0e28575270afdd9dcc0c13280a9b40445377d/chonkie_core-0.10.2-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:cb0626fa4d9df5128188eef069f69809571fbfc6db47e348d5f4d1199fa34b9c", size = 346802, upload-time = "2026-05-28T19:16:05.415Z" }, + { url = "https://files.pythonhosted.org/packages/26/4d/29e42ac094624a0ebfe74684ce62eb870e640679c1feb8da63e631a45cde/chonkie_core-0.10.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5266bdb8887c223782096c97a72eb49f19930f797213fa65b82199042742dc98", size = 336470, upload-time = "2026-05-28T19:16:06.575Z" }, + { url = "https://files.pythonhosted.org/packages/56/05/b3e206f063e6515a1b2e1f873ff33d91bf1eaab99554577c47804ebbbf37/chonkie_core-0.10.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4b6e2cf0411cb675d8b22f2b234b07e20724ded6cf84b2bc625b677885d2ef7", size = 386170, upload-time = "2026-05-28T19:16:07.761Z" }, + { url = "https://files.pythonhosted.org/packages/78/1e/4c775aec3d7ef6ff6ad90570c5b6f612fda9bf1ae7dfb4bd322d0fe55ffe/chonkie_core-0.10.2-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:d1999d993df8ecf941361b8eff9618f2a9bf3436b4a1851ec3f3f42d9a9c1a7e", size = 370624, upload-time = "2026-05-28T19:16:08.883Z" }, + { url = "https://files.pythonhosted.org/packages/f5/d0/103da313f8990607ac77c4735f372734a4d96afe79fdabc5883731bb2063/chonkie_core-0.10.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:acf83c774d5646dd9f16b896722155db8e87948fc93ee952fd649b5344d9205d", size = 101801, upload-time = "2026-05-28T19:16:10.226Z" }, + { url = "https://files.pythonhosted.org/packages/b0/93/e3167b1823f919f1a4c517092f73dc4107737de8bb53c67ff60c964b5244/chonkie_core-0.10.2-cp314-cp314-win_amd64.whl", hash = "sha256:20c9d64b9c5169d1f7e5ceeaa22bff3613017b8937c224ba85e3e24748fb0ba3", size = 228655, upload-time = "2026-05-28T19:16:11.357Z" }, +] + [[package]] name = "click" version = "8.3.1" @@ -1027,6 +1079,7 @@ dependencies = [ { name = "authlib" }, { name = "bcrypt" }, { name = "brotli-asgi" }, + { name = "chonkie" }, { name = "click" }, { name = "datasets" }, { name = "docling-core" }, @@ -1041,6 +1094,7 @@ dependencies = [ { name = "lancedb" }, { name = "lazy-loader" }, { name = "litellm" }, + { name = "liteparse" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "oauthlib" }, @@ -1104,6 +1158,7 @@ requires-dist = [ { name = "authlib", specifier = ">=1.6.9" }, { name = "bcrypt", specifier = ">=4.2.0" }, { name = "brotli-asgi", specifier = ">=1.4.0" }, + { name = "chonkie", specifier = ">=1.7.0" }, { name = "click", specifier = ">=8.2.0" }, { name = "datasets", specifier = ">=3.6.0" }, { name = "docling-core", specifier = ">=2.91.0,<3.0.0" }, @@ -1118,6 +1173,7 @@ requires-dist = [ { name = "lancedb", specifier = ">=0.37.1" }, { name = "lazy-loader", specifier = ">=0.4" }, { name = "litellm", specifier = ">=1.80.0,<=1.82.6" }, + { name = "liteparse", specifier = ">=2.14.0" }, { name = "numpy", specifier = ">=2.0.0,<3.0.0" }, { name = "oauthlib", specifier = ">=3.2.0" }, { name = "obstore", specifier = ">=0.11.0" }, @@ -1561,18 +1617,17 @@ wheels = [ [[package]] name = "httpx" -version = "0.27.2" +version = "0.28.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "certifi" }, { name = "httpcore" }, { name = "idna" }, - { name = "sniffio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/78/82/08f8c936781f67d9e6b9eeb8a0c8b4e406136ea4c3d1f89a5db71d42e0e6/httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2", size = 144189, upload-time = "2024-08-27T12:54:01.334Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/56/95/9377bcb415797e44274b51d46e3249eba641711cf3348050f76ee7b15ffc/httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0", size = 76395, upload-time = "2024-08-27T12:53:59.653Z" }, + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] [[package]] @@ -1881,6 +1936,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/53/aa31e4d057b3746b3c323ca993003d6cf15ef987e7fe7ceb53681695ae87/litellm-1.80.0-py3-none-any.whl", hash = "sha256:fd0009758f4772257048d74bf79bb64318859adb4ea49a8b66fdbc718cd80b6e", size = 10492975, upload-time = "2025-11-16T00:03:49.182Z" }, ] +[[package]] +name = "liteparse" +version = "2.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/91/b24e0bc63c3499a940c5b871203e32e40c4e08d7f5303705500725493e3a/liteparse-2.14.0.tar.gz", hash = "sha256:5f7295ab463353821faa897b3893ff2beb364a1d241551d8ba7707c679594e13", size = 488906, upload-time = "2026-08-25T23:32:20.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/98/735ad3d97e5cf4285ade684f0830511a365dc2702935079ad0b80691d70c/liteparse-2.14.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c36e614b7cc5dbda00b3bd65e128bac9fc4ccaf12cde1d1ad38741fd2116259a", size = 12568724, upload-time = "2026-08-25T23:32:04.429Z" }, + { url = "https://files.pythonhosted.org/packages/e6/eb/aa1f43ce2013a58e795f2dfe5ba92849e6b74ac93fcd2913dd847e0c1db0/liteparse-2.14.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:1896019aa736aac5f6594dd9433d1a15da5c9d868c75826ec97f470e987d33b7", size = 11971374, upload-time = "2026-08-25T23:32:06.841Z" }, + { url = "https://files.pythonhosted.org/packages/fb/20/727c2516e8626a8c83a6de46b2edbb7d7b2975edd5ddc39426ee11d82e6d/liteparse-2.14.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7219c9f9df82d46e1f29e56c7aa90f965c40067c92032d9d543f8b9c48b600ce", size = 13495316, upload-time = "2026-08-25T23:32:08.896Z" }, + { url = "https://files.pythonhosted.org/packages/d1/e0/c472f229287e4d4c1617ae07aeb2f8cc35f3bd99af7a7f8b17c87ff4a537/liteparse-2.14.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:91a6794b53425010a506cd8403a953c13e006039b593d4ab0cd36ad812e2cb9e", size = 13792828, upload-time = "2026-08-25T23:32:10.896Z" }, + { url = "https://files.pythonhosted.org/packages/e7/fd/7f0e19553301a56e24a413f5c2a0d8045bfd74ac2e870bf6b6aa58fc0da1/liteparse-2.14.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:90340ba3467df9f816b5cb866f7041b034d0da6de83559707795bd4edf5ea13d", size = 16968115, upload-time = "2026-08-25T23:32:12.938Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a9/f748a731a4b62f0d839ae03f4b008f81e5a1e2a210524648884b4114d59d/liteparse-2.14.0-cp310-abi3-win_amd64.whl", hash = "sha256:a864a5c939ca8061c9ecb02a7f80dde3cda452ec64394064f9514aee4b9de97a", size = 11996465, upload-time = "2026-08-25T23:32:15.114Z" }, + { url = "https://files.pythonhosted.org/packages/77/77/5995104db59e9d34d9158c27623ca92a8d6b43cf28fdf5aa0d3b4b86e556/liteparse-2.14.0-cp310-abi3-win_arm64.whl", hash = "sha256:28bcc45df09cacf9d75e946b8ccd21c48021912802835ec2375fdc16a185973b", size = 11306309, upload-time = "2026-08-25T23:32:17.563Z" }, +] + [[package]] name = "lxml" version = "6.0.2" @@ -4142,6 +4212,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/59/8c/b1c87148aa15e099243ec9f0cf9d0e970cc2234c3257d558c25a2c5304e6/tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f01a9c019878532f98927d2bacb79bbb404b43d3437455522a00a30718cdedb5", size = 3373542, upload-time = "2026-01-05T10:40:52.803Z" }, ] +[[package]] +name = "tokie" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/7e/ccb0f35fcdaea03ea4db7b4f6066aeec63d71bebf83d8181cda2fa320661/tokie-0.1.4.tar.gz", hash = "sha256:c8707df026d79a13228d410d607fde0dff5860d0f45ce11d89d85fb0a4c2a114", size = 280777, upload-time = "2026-07-24T16:41:15.502Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/f8/6507e5ef1fa3ccf8d8b453787ae5e8dd3214bdf925194442adad9b72e71c/tokie-0.1.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:80ff46843015605d2c3e34889f68a370d41d0f257e0edee4a4c253916b9eb3ba", size = 2866087, upload-time = "2026-07-24T16:40:35.314Z" }, + { url = "https://files.pythonhosted.org/packages/d0/3e/2b23f018a1bced6603a049c2c73b6bf902da17a5956863847b81b06c2133/tokie-0.1.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:299a87141ad6802fb48c7146dae3496c3716fc54f99e2a909b32c742780fa03a", size = 2897113, upload-time = "2026-07-24T16:40:37.475Z" }, + { url = "https://files.pythonhosted.org/packages/94/b0/c3bb8317d854c6da6a56f646bfd99ef7632c71b837462f06c8af15dbbebf/tokie-0.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:450e7270206afad356cad1e7d9c83cda98e5a0a8e633957df0abc484ddc08589", size = 3095044, upload-time = "2026-07-24T16:40:38.995Z" }, + { url = "https://files.pythonhosted.org/packages/3e/f5/839989186caa2769936f1e0a5dac7d33e6d73a46a958a164f8292cecc887/tokie-0.1.4-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:440c992d28f457f79ff244f1b8291fa0e245d058f131619dee728b99ba7e5eb8", size = 3207176, upload-time = "2026-07-24T16:40:40.588Z" }, + { url = "https://files.pythonhosted.org/packages/5b/0e/5c6cf9a817513dcc081c922a333b0a54c2d9ad506d32ba2d31a1f864e40d/tokie-0.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:0bc02cdd9bd9a15d997a7b5f0beb3e148d9c3efd5d949b9f51077f6c53aef6d1", size = 2639034, upload-time = "2026-07-24T16:40:42.367Z" }, + { url = "https://files.pythonhosted.org/packages/19/b6/53cd60a1182ce475812ae38ecaff15cfba4b447976920289f2caf471b14c/tokie-0.1.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4d922e3ba6635ecd513cfd6b8cb0969e88c0b329a41cbc0dd0c1a2c9053800f0", size = 2865741, upload-time = "2026-07-24T16:40:43.852Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b8/cf7a51c9d08bd2e5f4f8510c4da20b558bb8056e215a6045333308c07f69/tokie-0.1.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d06278ae88011f0fc6ff99d36707c856f633131f94dfc35f2033d0a71107d4bf", size = 2897068, upload-time = "2026-07-24T16:40:45.457Z" }, + { url = "https://files.pythonhosted.org/packages/b0/04/e447decd69987e3f4c00015ed4cd08106d851fb9ab2e9f34643d5cff7df4/tokie-0.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8a5b5dd143d807981711e973a9fff21a4fc48e472ef3a976054be04ac236a63", size = 3094709, upload-time = "2026-07-24T16:40:46.855Z" }, + { url = "https://files.pythonhosted.org/packages/de/d9/3b10e0c2c753db3b85a55d891548796f2fc16e5000941167ff2c6f3c6428/tokie-0.1.4-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d3ef8dcf454b1601d80ef8cfb5c5b86f89c8dd26f68dd05a5c25248872d20039", size = 3206948, upload-time = "2026-07-24T16:40:48.266Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/41aeae9d70bb7484ca4e4a9166d5330781b4a59b3cb8f1413a6b62e28204/tokie-0.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:885691c40fe47b1daaff1e7c9d13cc2f9734c8cd089a00fee5d05ee0f9a703d0", size = 2638905, upload-time = "2026-07-24T16:40:49.904Z" }, + { url = "https://files.pythonhosted.org/packages/7b/af/fae60b61803429d29f0ab5d01e474ff8461496d91c6c7753a7cede5d8db7/tokie-0.1.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3105e5d0df6829fddc0a6761e726dcae80cb8b373f39fe6396a32a2014157478", size = 2865113, upload-time = "2026-07-24T16:40:52.305Z" }, + { url = "https://files.pythonhosted.org/packages/4a/3f/ee12c8b77393ce6ceb5c465908b43ab96af4499fff88bff7e16f13022b09/tokie-0.1.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4ad390232d70e8e4b239e6e7edd8578e56433014ad496ea3597faa677b5e18b3", size = 2889620, upload-time = "2026-07-24T16:40:53.665Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f4/32eaa1aa37a5ad86244a1315096401cccefbb82c2c723b7181a42ee668e0/tokie-0.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eecd300fa19704fb3022aaedf8c1eafa9d9825c87ee13d2203e383bab4e764f", size = 3089894, upload-time = "2026-07-24T16:40:55.126Z" }, + { url = "https://files.pythonhosted.org/packages/62/66/b630a22d546da4712765e0472c4dbfdf50dcfc77b821ad274b12ee7be469/tokie-0.1.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:bbab9991a569beb667a0fe5f7c5698050914a46c015d0d323d8f4ef9e8c8aee2", size = 3202845, upload-time = "2026-07-24T16:40:56.571Z" }, + { url = "https://files.pythonhosted.org/packages/4c/64/e2bf4606adb89c942f72524b2a5a832cb6899676a623df04adaea1eee5b0/tokie-0.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:b0d245887bb806c42593304abce545fd1e7808c091946a9f3923d9211e99f4ff", size = 2635997, upload-time = "2026-07-24T16:40:58.283Z" }, + { url = "https://files.pythonhosted.org/packages/96/15/3fe8af5fe7ee607a6f6817e1119000b8bc828c9322dd68cccc724334ce38/tokie-0.1.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:e28c1c825479c8783ef2917bcc75964113d16ece254f9cd83234d19f3eaf7ff3", size = 2864983, upload-time = "2026-07-24T16:40:59.627Z" }, + { url = "https://files.pythonhosted.org/packages/ea/3f/35246c2ea85eb773342c64521da2481b0ff363cf1b39b1118eb6ab1c2ce3/tokie-0.1.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:199738e086f4a32892e651d9f98d10e606d12d581563041183e5c1b37a83745b", size = 2890250, upload-time = "2026-07-24T16:41:01.144Z" }, + { url = "https://files.pythonhosted.org/packages/f0/6c/114dfa287748d1bc32a0ff2ea56c24284a8afa4b9a1a391c65d2b20fed90/tokie-0.1.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2c41b8565e22aa0692e4cfcab595c0fe8b7ee742874e884ab9df928aa16d70e", size = 3089343, upload-time = "2026-07-24T16:41:02.629Z" }, + { url = "https://files.pythonhosted.org/packages/82/45/3e0fb358bd6fc3a9166cb7472f48055d7e894188b6f7553ebbad9e4e81f6/tokie-0.1.4-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:b39bf238b7b89bce19fa8ee353581ecffed46a7aac038502b9649789854b7e51", size = 3202698, upload-time = "2026-07-24T16:41:04.36Z" }, + { url = "https://files.pythonhosted.org/packages/54/99/d2444072b3ad7f2bcb7c40286e71d39f1cb968ebc52a90e6de31d5e5782d/tokie-0.1.4-cp313-cp313-win_amd64.whl", hash = "sha256:3ded2223b0486c8cf7922b9cf632ff10ee334bb7c736094f257515115826de87", size = 2635505, upload-time = "2026-07-24T16:41:05.533Z" }, + { url = "https://files.pythonhosted.org/packages/0f/32/4a0f5a30c5f31868fe795f6230cfad64545ed4900ee98f9f16f688f9dbb7/tokie-0.1.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:29013b3b7b6a6f3ed4b2e1fe35eeb4702236088c59bf137f8cb85954d69d5e84", size = 2866161, upload-time = "2026-07-24T16:41:06.973Z" }, + { url = "https://files.pythonhosted.org/packages/e2/db/150c266d98d5093fb65c9d150c8594d598a2b2626e6c4b88907552659d00/tokie-0.1.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b80085348b77d7e783513bfdc79dd4452c0602a28e04907124dcc17cbed8e5", size = 2890795, upload-time = "2026-07-24T16:41:08.397Z" }, + { url = "https://files.pythonhosted.org/packages/4c/42/078cc772347e8b6279fd9b643b5c7fb47eb49fe4eef88e3b8bbe165bbffc/tokie-0.1.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25344e4c02e47fc8cd2a146b1aa4ea5286b478bcbc5b109b1ec1da8d0ac17ff1", size = 3090930, upload-time = "2026-07-24T16:41:09.851Z" }, + { url = "https://files.pythonhosted.org/packages/6b/3f/8267161d70197f25ebe33e01907cac6a427228357b4ff9653900020f0e1a/tokie-0.1.4-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:cbc7b468fba5d5f506b85bb4319ffe825ff2c328aaea3c8ff57a36740af00e6d", size = 3203012, upload-time = "2026-07-24T16:41:11.375Z" }, + { url = "https://files.pythonhosted.org/packages/78/36/b97834b2152c24e6c5a5aaa39b6e4659105b3b3c472877e354d19baf3237/tokie-0.1.4-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:9b90d2ac37476497b7f5de25df4208b83d6a4a3cf43c790b7b91dcbc5419da8b", size = 958833, upload-time = "2026-07-24T16:41:12.715Z" }, + { url = "https://files.pythonhosted.org/packages/85/9d/d727a3a0d05bb78cc839e77a46b00bf8542769384ac6aa64c953314d5bfc/tokie-0.1.4-cp314-cp314-win_amd64.whl", hash = "sha256:2d4fb5a6b9cd4e137c3e6429129a5901909414552f35236c213668fbeab6cc77", size = 2637400, upload-time = "2026-07-24T16:41:14.288Z" }, +] + [[package]] name = "tomli" version = "2.4.0"