From 30a08df8405b6b2ac75ca582f7563fff5a141744 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Wed, 26 Aug 2026 00:23:15 -0700 Subject: [PATCH 01/16] build(server): add liteparse + chonkie, bake the retrieval payloads into the image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 0 of the retrieval plan. liteparse 2.14 ships cp310-abi3 wheels for both manylinux arches, so the block extractor no longer pins a single CPython — the 3.12 default from 270efb1 stays, but requires-python >=3.10 remains honest. The image now carries the two things a first request must not have to fetch: `tesseract-ocr-eng` (4 MB) for liteparse's OCR path and the DuckDB `lance` community extension (231 MB), installed as `extralit` so it lands in the home DuckDB resolves against. `smoke_retrieval_deps.py` asserts all four in CI right after the existing CLI check; image grows 1.55 GB -> 1.8 GB, 92% of it the extension binary. chonkie requires httpx>=0.28.1, which removes `AsyncClient(app=...)` and encodes `json=` with allow_nan=False. Both conftests move to `ASGITransport`, and the three tests that deliberately post NaN metadata now send it as `content=` — otherwise the client rejects the payload the server is supposed to 422 on. --- .../extralit-server.build-docker-images.yml | 4 + extralit-server/docker/server/Dockerfile | 11 +- .../server/scripts/smoke_retrieval_deps.py | 73 ++++++++++++ extralit-server/pyproject.toml | 3 + extralit-server/tests/integration/conftest.py | 4 +- .../unit/api/handlers/v1/test_datasets.py | 44 ++++--- extralit-server/tests/unit/conftest.py | 4 +- extralit-server/uv.lock | 112 +++++++++++++++++- 8 files changed, 227 insertions(+), 28 deletions(-) create mode 100644 extralit-server/docker/server/scripts/smoke_retrieval_deps.py diff --git a/.github/workflows/extralit-server.build-docker-images.yml b/.github/workflows/extralit-server.build-docker-images.yml index 3ce6a58e1..ca3e3d458 100644 --- a/.github/workflows/extralit-server.build-docker-images.yml +++ b/.github/workflows/extralit-server.build-docker-images.yml @@ -125,6 +125,10 @@ jobs: run: | docker run --rm ${{ env.SERVER_DOCKER_IMAGE }}:${{ env.IMAGE_TAG }} python -m extralit_server start --help + - name: Check the retrieval payloads are baked into the image + run: | + docker run --rm ${{ env.SERVER_DOCKER_IMAGE }}:${{ env.IMAGE_TAG }} python smoke_retrieval_deps.py + - name: Push latest `extralit-server` image if: ${{ env.PUBLISH_LATEST == 'true' }} uses: docker/build-push-action@v5 diff --git a/extralit-server/docker/server/Dockerfile b/extralit-server/docker/server/Dockerfile index 0a508e4dd..47c965d7a 100644 --- a/extralit-server/docker/server/Dockerfile +++ b/extralit-server/docker/server/Dockerfile @@ -39,12 +39,15 @@ RUN mkdir -p "$EXTRALIT_HOME_PATH" && \ chown extralit:extralit "$EXTRALIT_HOME_PATH" && \ apt-get update && \ apt-get upgrade -y && \ - apt-get install -y --no-install-recommends libpq-dev libgl1 libglib2.0-0 poppler-utils && \ + apt-get install -y --no-install-recommends libpq-dev libgl1 libglib2.0-0 poppler-utils tesseract-ocr-eng && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* VOLUME $EXTRALIT_HOME_PATH -COPY scripts/start_extralit_server.sh /home/extralit +# liteparse would otherwise download the language data on the first scanned page it meets. +ENV TESSDATA_PREFIX=/usr/share/tesseract-ocr/5/tessdata + +COPY scripts/start_extralit_server.sh scripts/smoke_retrieval_deps.py /home/extralit/ # Destination folder must be the same as the builder one. Otherwise installed script won't work (since the installation fixes the path inside the script) COPY --chown=extralit:extralit --from=builder /opt/venv /opt/venv @@ -57,6 +60,10 @@ RUN chmod +x start_extralit_server.sh USER extralit +# Pull the community `lance` extension now: DuckDB resolves it per user home, and a hybrid +# search must not be the thing that discovers the runtime has no network. +RUN python -c "import duckdb; duckdb.connect().execute('INSTALL lance')" + # Exposing ports EXPOSE 6900 diff --git a/extralit-server/docker/server/scripts/smoke_retrieval_deps.py b/extralit-server/docker/server/scripts/smoke_retrieval_deps.py new file mode 100644 index 000000000..21a1d0c4c --- /dev/null +++ b/extralit-server/docker/server/scripts/smoke_retrieval_deps.py @@ -0,0 +1,73 @@ +"""Assert the retrieval pipeline's non-Python payloads are present in this environment. + + docker run --rm extralit/extralit-server:latest python smoke_retrieval_deps.py + +Each of these fails late and confusingly in production — a first hybrid search that cannot +reach the DuckDB extension repository, or a first scanned page that finds no language data — +so the image build runs this instead of discovering them under load. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + + +def check_lance_extension() -> str: + import duckdb + + connection = duckdb.connect() + # No INSTALL: the point is that the extension is already on disk. + connection.execute("LOAD lance") + functions = connection.execute( + "select function_name from duckdb_functions() where function_name in ('lance_fts', 'lance_vector_search')" + ).fetchall() + missing = {"lance_fts", "lance_vector_search"} - {name for (name,) in functions} + if missing: + raise RuntimeError(f"the lance extension loaded without {sorted(missing)}") + return f"duckdb {duckdb.__version__} + lance extension" + + +def check_liteparse() -> str: + import liteparse + + parser = liteparse.LiteParse(extract_blocks=True, quiet=True) + parser.close() + return f"liteparse {liteparse.__version__}" + + +def check_tessdata() -> str: + prefix = os.environ.get("TESSDATA_PREFIX") + if not prefix: + raise RuntimeError("TESSDATA_PREFIX is unset, so OCR would download language data on first use") + if not (Path(prefix) / "eng.traineddata").is_file(): + raise RuntimeError(f"no eng.traineddata under {prefix}") + return f"tessdata at {prefix}" + + +def check_chonkie() -> str: + from chonkie import RecursiveChunker, RecursiveRules + + chunker = RecursiveChunker(tokenizer="character", chunk_size=64, rules=RecursiveRules()) + if not chunker("one two three. four five six."): + raise RuntimeError("RecursiveChunker returned no chunks") + import chonkie + + return f"chonkie {chonkie.__version__}" + + +def main() -> int: + failures = 0 + for check in (check_lance_extension, check_liteparse, check_tessdata, check_chonkie): + name = check.__name__.removeprefix("check_") + try: + print(f"ok {name}: {check()}") + except Exception as error: + failures += 1 + print(f"FAIL {name}: {type(error).__name__}: {error}", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/extralit-server/pyproject.toml b/extralit-server/pyproject.toml index dcb48fd3d..29a1ea3c8 100644 --- a/extralit-server/pyproject.toml +++ b/extralit-server/pyproject.toml @@ -80,6 +80,9 @@ dependencies = [ "docling-core>=2.91.0,<3.0.0", "pyarrow>=23.0.1", "pdf-inspector>=1.14.2", + # abi3 wheels from 2.14, so the block extractor is not tied to one CPython + "liteparse>=2.14.0", + "chonkie>=1.7.0", "xxhash>=3.6.0", "obstore>=0.11.0", ] diff --git a/extralit-server/tests/integration/conftest.py b/extralit-server/tests/integration/conftest.py index affdadb64..96fb82002 100644 --- a/extralit-server/tests/integration/conftest.py +++ b/extralit-server/tests/integration/conftest.py @@ -10,7 +10,7 @@ import pytest import pytest_asyncio -from httpx import AsyncClient +from httpx import ASGITransport, AsyncClient from extralit_server.constants import API_KEY_HEADER_NAME from extralit_server.database import get_async_db @@ -39,7 +39,7 @@ async def override_get_async_db(): api_v1.dependency_overrides[get_async_db] = override_get_async_db - async with AsyncClient(app=app, base_url="http://testserver") as client: + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://testserver") as client: yield client # Pop only what this fixture registered. `tests/unit/conftest.py`'s async_client writes diff --git a/extralit-server/tests/unit/api/handlers/v1/test_datasets.py b/extralit-server/tests/unit/api/handlers/v1/test_datasets.py index 1b0603135..02d23a6aa 100644 --- a/extralit-server/tests/unit/api/handlers/v1/test_datasets.py +++ b/extralit-server/tests/unit/api/handlers/v1/test_datasets.py @@ -1,3 +1,4 @@ +import json import math import uuid from datetime import datetime @@ -82,6 +83,11 @@ from sqlalchemy.ext.asyncio import AsyncSession +def nan_body(payload: dict, headers: dict) -> dict: + """Post a body the JSON spec forbids. `json=` cannot: httpx encodes with allow_nan=False.""" + return {"content": json.dumps(payload), "headers": {**headers, "Content-Type": "application/json"}} + + @pytest.mark.asyncio class TestSuiteDatasets: async def test_list_current_user_datasets(self, async_client: "AsyncClient", owner_auth_header: dict) -> None: @@ -1959,7 +1965,7 @@ async def test_create_dataset_records_with_metadata_nan_values( } response = await async_client.post( - f"/api/v1/datasets/{dataset.id}/records/bulk", headers=owner_auth_header, json=records_json + f"/api/v1/datasets/{dataset.id}/records/bulk", **nan_body(records_json, owner_auth_header) ) assert response.status_code == 422 @@ -3027,23 +3033,25 @@ async def test_update_dataset_records_with_metadata_nan_value( response = await async_client.put( f"/api/v1/datasets/{dataset.id}/records/bulk", - headers=owner_auth_header, - json={ - "items": [ - { - "id": str(records[0].id), - "metadata": {"terms": math.nan}, - }, - { - "id": str(records[1].id), - "metadata": {"float": math.nan}, - }, - { - "id": str(records[2].id), - "metadata": {"terms": "a"}, - }, - ] - }, + **nan_body( + { + "items": [ + { + "id": str(records[0].id), + "metadata": {"terms": math.nan}, + }, + { + "id": str(records[1].id), + "metadata": {"float": math.nan}, + }, + { + "id": str(records[2].id), + "metadata": {"terms": "a"}, + }, + ] + }, + owner_auth_header, + ), ) assert response.status_code == 422 diff --git a/extralit-server/tests/unit/conftest.py b/extralit-server/tests/unit/conftest.py index 8d7c934c3..5bc473561 100644 --- a/extralit-server/tests/unit/conftest.py +++ b/extralit-server/tests/unit/conftest.py @@ -3,7 +3,7 @@ import pytest import pytest_asyncio -from httpx import AsyncClient +from httpx import ASGITransport, AsyncClient from opensearchpy import OpenSearch from sqlalchemy.engine.interfaces import IsolationLevel @@ -95,7 +95,7 @@ async def override_get_search_engine(): } ) - async with AsyncClient(app=app, base_url="http://testserver") as async_client: + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://testserver") as async_client: yield async_client # Clear from `api_v1` — that is where the overrides above were registered. Clearing diff --git a/extralit-server/uv.lock b/extralit-server/uv.lock index c33ac8487..16c719caa 100644 --- a/extralit-server/uv.lock +++ b/extralit-server/uv.lock @@ -608,6 +608,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/68/687187c7e26cb24ccbd88e5069f5ef00eba804d36dde11d99aad0838ab45/charset_normalizer-3.4.6-py3-none-any.whl", hash = "sha256:947cf925bc916d90adba35a64c82aace04fa39b46b52d4630ece166655905a69", size = 61455, upload-time = "2026-03-15T18:53:23.833Z" }, ] +[[package]] +name = "chonkie" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "chonkie-core" }, + { name = "httpx" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "tenacity" }, + { name = "tokie" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/67/e6ceace2b2066cc38206c81e9105735975d7a58d89c9e7a871fe9acef63d/chonkie-1.7.0.tar.gz", hash = "sha256:4352aa214b66376c32523e524b94b7f1456a5e7611c48037dadb45e46f436b18", size = 190834, upload-time = "2026-07-07T10:06:02.475Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/c0/b5f91fb340d354a8c20e2f65d08b9dab357360da78d230e96c6d79b36a2e/chonkie-1.7.0-py3-none-any.whl", hash = "sha256:98822ca80fbf3c0775f59329a4f6b6bfedb34edff004b5e04e53c3281369b921", size = 233764, upload-time = "2026-07-07T10:06:00.824Z" }, +] + +[[package]] +name = "chonkie-core" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/17/ad31bbcfe1a7b63f76e065684a8e9cd98552d3c627fb28d8fbc05f1a9456/chonkie_core-0.10.2.tar.gz", hash = "sha256:c8e40ef8f3a034a7c5dd23a0401dce2ef2b4883f5a6a29cf94176d64b209bdbb", size = 69966, upload-time = "2026-05-28T19:16:12.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/a0/16ff6f6d3ddfb3f39d89663717e91eaa1bd5ba6ac52ec2fa01795ac33c65/chonkie_core-0.10.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b63d25a3270a98eb5a6739072315f36566c87139078797a0970bc8d480f5bbe0", size = 349233, upload-time = "2026-05-28T19:15:38.805Z" }, + { url = "https://files.pythonhosted.org/packages/ed/9d/2ed7af6666a168fcb18328e06354035f506aa8965a479827aa6f5528b189/chonkie_core-0.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:86c3e04603ce69d380f3f6a8f14f85dd902410ad1ec86709ddcfad43dc968946", size = 339167, upload-time = "2026-05-28T19:15:40.388Z" }, + { url = "https://files.pythonhosted.org/packages/12/bd/373ca97e84a4c4caf9adba9e710bc48f50c87524379a949cd17ecc588def/chonkie_core-0.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2225d6e6622b26d4542ed9661e71b51dfd8b4c5ae7e652fe85b4f0faa4be8eba", size = 390116, upload-time = "2026-05-28T19:15:41.556Z" }, + { url = "https://files.pythonhosted.org/packages/09/62/c3ae371065d31e52eb96d05ef02384521c99dd6bd036bc4b1d7a09ea4cbb/chonkie_core-0.10.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:e7e05701463e8e7a339a7be617fe2306153dcb5f269f43251e1516f40ce91134", size = 375235, upload-time = "2026-05-28T19:15:42.763Z" }, + { url = "https://files.pythonhosted.org/packages/16/08/3f55dbb5cd033b9c33e9e5fbaa5ee88af3dbc78e6858cf42341c903c5cc6/chonkie_core-0.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:c7c2c431d86fbae7c1f8d0b8628236df27b4dab040dc6504ded209be9e36f5b0", size = 230868, upload-time = "2026-05-28T19:15:44.207Z" }, + { url = "https://files.pythonhosted.org/packages/c9/0b/4c28270b24d9e756c90a52966f26ddff5fa2f639aed7e6cd9cc1e1728176/chonkie_core-0.10.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a431af90ac4e4072e02699ac1300efcaaff2ee432b052fe51027998859af1055", size = 349504, upload-time = "2026-05-28T19:15:45.801Z" }, + { url = "https://files.pythonhosted.org/packages/93/a6/069b46520a23264585ddacd117bee62594afff0f851cd1dfedf32b900faf/chonkie_core-0.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:712ecc448302bdc8f3ab4aab1e17d5d37b50d57d89d4fa4ec44821a37eb8fa57", size = 339673, upload-time = "2026-05-28T19:15:46.992Z" }, + { url = "https://files.pythonhosted.org/packages/a5/05/513baf0c159fb93e6c05bf199cca79045b2c186e9fea1fd7ebf47d063c7b/chonkie_core-0.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff79cbe5016f85a8fac2164e1716f3335b631704bfa486f0d1de02c6d88c2bce", size = 390030, upload-time = "2026-05-28T19:15:48.434Z" }, + { url = "https://files.pythonhosted.org/packages/50/65/4a86e1648147df94bcc2d681b6c5f40a21c57df65b45bf2f3ceb72fcb784/chonkie_core-0.10.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:f69969fa3f75930bd10f1ec502993b758df4e5a62d170c373a4fa73743dc2f93", size = 374622, upload-time = "2026-05-28T19:15:49.836Z" }, + { url = "https://files.pythonhosted.org/packages/54/62/425bca737db62a371b1e3393d76f69f483baf3482ccd88504089501ee0e4/chonkie_core-0.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:cbe5a8a1e89a79a74bac99409e1bf5e7a5b3e2286ac5f2185077d39f5fa95175", size = 231028, upload-time = "2026-05-28T19:15:50.956Z" }, + { url = "https://files.pythonhosted.org/packages/56/2f/4dd88b4af9ef0e9ed31a3c6a3d8e44327c5ca22b074076830b0224e0ad58/chonkie_core-0.10.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:31f8fa3415d2e93cc3bdb1e99c3b6865278169af348c68fb2943aed2fcabc010", size = 347520, upload-time = "2026-05-28T19:15:52.331Z" }, + { url = "https://files.pythonhosted.org/packages/b2/3f/ba4083f21b4ef52ed130a79f371fed453b7edf998bf08079694137880bff/chonkie_core-0.10.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a6d081363fbfaad0f36c66e67e6dec1c9f9850c04c0475b64b3e47084257b646", size = 336897, upload-time = "2026-05-28T19:15:53.787Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9f/cbfec0f25e35d2a5b62a9cba7a5a85cfc5349210b9f9acb40832060211e4/chonkie_core-0.10.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51dbf2163dac5ad695dde3f92155c6febb9332b5fb5a174ac80ae9fc8f8ccde9", size = 387231, upload-time = "2026-05-28T19:15:54.851Z" }, + { url = "https://files.pythonhosted.org/packages/89/c8/b0d91419730bff4566305f2c85f27fb1378e8e1e99f8e59403c6b25d43a2/chonkie_core-0.10.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5a09bcb56d9d9f2fa6ec84485aeecbcaa375567eea9fbec18517d2ef643b9029", size = 371506, upload-time = "2026-05-28T19:15:56.172Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c2/05c2c85b11d092f04768c41fb9cfdd27a9d27fa7e10e107188a51ce8813f/chonkie_core-0.10.2-cp312-cp312-win_amd64.whl", hash = "sha256:ac686d192c4dd7e038cea4aba59c677d45e2b1e8de8eb1dd45d269b22a4ae201", size = 229686, upload-time = "2026-05-28T19:15:57.759Z" }, + { url = "https://files.pythonhosted.org/packages/59/4f/0814d2bd9abddbbc37f8c204f056d28f2ab7d14bf742515e9684062bd758/chonkie_core-0.10.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:25372879e43b235dee01f48309634a3fce07b55c382a7fbe200d6b66769267e4", size = 346848, upload-time = "2026-05-28T19:15:58.979Z" }, + { url = "https://files.pythonhosted.org/packages/3f/33/b640f406b6d2fe30d5c41d4a91411d9ed1dc22acf6497cda837a6f121ba4/chonkie_core-0.10.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3da60e797b2fcbef3e17fad3e46bb7d4f5541ac5f87a5556c2c610cef83a549a", size = 336613, upload-time = "2026-05-28T19:16:00.125Z" }, + { url = "https://files.pythonhosted.org/packages/e9/76/079296e83d2ed6e34b8ed24bd23e59193b502b53ddfa176b3e921410a5db/chonkie_core-0.10.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aebeba2d2cb0d30eefdbdb18d2ab2fcfbc0ea0d5121f61f537a6114a1da56101", size = 386412, upload-time = "2026-05-28T19:16:01.425Z" }, + { url = "https://files.pythonhosted.org/packages/04/6e/2c505276878b695b4cee07eb35a5c536ed92d60c722b4547da1517fb3e41/chonkie_core-0.10.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:72974163eab058d14c365f45b53d2367b1e04a0dcd779dedd828973cf21256d7", size = 371726, upload-time = "2026-05-28T19:16:02.8Z" }, + { url = "https://files.pythonhosted.org/packages/38/39/158f85b728d84e85a00768ccebe3a289d21462f52ba7850db9e01e61e4df/chonkie_core-0.10.2-cp313-cp313-win_amd64.whl", hash = "sha256:9f77811bb722bd019a52353364bc6dcea1a9998120a413f1c2a1c79153a66386", size = 229447, upload-time = "2026-05-28T19:16:04.21Z" }, + { url = "https://files.pythonhosted.org/packages/0b/45/e4d68840847133afa9c69fc0e28575270afdd9dcc0c13280a9b40445377d/chonkie_core-0.10.2-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:cb0626fa4d9df5128188eef069f69809571fbfc6db47e348d5f4d1199fa34b9c", size = 346802, upload-time = "2026-05-28T19:16:05.415Z" }, + { url = "https://files.pythonhosted.org/packages/26/4d/29e42ac094624a0ebfe74684ce62eb870e640679c1feb8da63e631a45cde/chonkie_core-0.10.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5266bdb8887c223782096c97a72eb49f19930f797213fa65b82199042742dc98", size = 336470, upload-time = "2026-05-28T19:16:06.575Z" }, + { url = "https://files.pythonhosted.org/packages/56/05/b3e206f063e6515a1b2e1f873ff33d91bf1eaab99554577c47804ebbbf37/chonkie_core-0.10.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4b6e2cf0411cb675d8b22f2b234b07e20724ded6cf84b2bc625b677885d2ef7", size = 386170, upload-time = "2026-05-28T19:16:07.761Z" }, + { url = "https://files.pythonhosted.org/packages/78/1e/4c775aec3d7ef6ff6ad90570c5b6f612fda9bf1ae7dfb4bd322d0fe55ffe/chonkie_core-0.10.2-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:d1999d993df8ecf941361b8eff9618f2a9bf3436b4a1851ec3f3f42d9a9c1a7e", size = 370624, upload-time = "2026-05-28T19:16:08.883Z" }, + { url = "https://files.pythonhosted.org/packages/f5/d0/103da313f8990607ac77c4735f372734a4d96afe79fdabc5883731bb2063/chonkie_core-0.10.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:acf83c774d5646dd9f16b896722155db8e87948fc93ee952fd649b5344d9205d", size = 101801, upload-time = "2026-05-28T19:16:10.226Z" }, + { url = "https://files.pythonhosted.org/packages/b0/93/e3167b1823f919f1a4c517092f73dc4107737de8bb53c67ff60c964b5244/chonkie_core-0.10.2-cp314-cp314-win_amd64.whl", hash = "sha256:20c9d64b9c5169d1f7e5ceeaa22bff3613017b8937c224ba85e3e24748fb0ba3", size = 228655, upload-time = "2026-05-28T19:16:11.357Z" }, +] + [[package]] name = "click" version = "8.3.1" @@ -1027,6 +1079,7 @@ dependencies = [ { name = "authlib" }, { name = "bcrypt" }, { name = "brotli-asgi" }, + { name = "chonkie" }, { name = "click" }, { name = "datasets" }, { name = "docling-core" }, @@ -1041,6 +1094,7 @@ dependencies = [ { name = "lancedb" }, { name = "lazy-loader" }, { name = "litellm" }, + { name = "liteparse" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "oauthlib" }, @@ -1104,6 +1158,7 @@ requires-dist = [ { name = "authlib", specifier = ">=1.6.9" }, { name = "bcrypt", specifier = ">=4.2.0" }, { name = "brotli-asgi", specifier = ">=1.4.0" }, + { name = "chonkie", specifier = ">=1.7.0" }, { name = "click", specifier = ">=8.2.0" }, { name = "datasets", specifier = ">=3.6.0" }, { name = "docling-core", specifier = ">=2.91.0,<3.0.0" }, @@ -1118,6 +1173,7 @@ requires-dist = [ { name = "lancedb", specifier = ">=0.37.1" }, { name = "lazy-loader", specifier = ">=0.4" }, { name = "litellm", specifier = ">=1.80.0,<=1.82.6" }, + { name = "liteparse", specifier = ">=2.14.0" }, { name = "numpy", specifier = ">=2.0.0,<3.0.0" }, { name = "oauthlib", specifier = ">=3.2.0" }, { name = "obstore", specifier = ">=0.11.0" }, @@ -1561,18 +1617,17 @@ wheels = [ [[package]] name = "httpx" -version = "0.27.2" +version = "0.28.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "certifi" }, { name = "httpcore" }, { name = "idna" }, - { name = "sniffio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/78/82/08f8c936781f67d9e6b9eeb8a0c8b4e406136ea4c3d1f89a5db71d42e0e6/httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2", size = 144189, upload-time = "2024-08-27T12:54:01.334Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/56/95/9377bcb415797e44274b51d46e3249eba641711cf3348050f76ee7b15ffc/httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0", size = 76395, upload-time = "2024-08-27T12:53:59.653Z" }, + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] [[package]] @@ -1881,6 +1936,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/53/aa31e4d057b3746b3c323ca993003d6cf15ef987e7fe7ceb53681695ae87/litellm-1.80.0-py3-none-any.whl", hash = "sha256:fd0009758f4772257048d74bf79bb64318859adb4ea49a8b66fdbc718cd80b6e", size = 10492975, upload-time = "2025-11-16T00:03:49.182Z" }, ] +[[package]] +name = "liteparse" +version = "2.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/91/b24e0bc63c3499a940c5b871203e32e40c4e08d7f5303705500725493e3a/liteparse-2.14.0.tar.gz", hash = "sha256:5f7295ab463353821faa897b3893ff2beb364a1d241551d8ba7707c679594e13", size = 488906, upload-time = "2026-08-25T23:32:20.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/98/735ad3d97e5cf4285ade684f0830511a365dc2702935079ad0b80691d70c/liteparse-2.14.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c36e614b7cc5dbda00b3bd65e128bac9fc4ccaf12cde1d1ad38741fd2116259a", size = 12568724, upload-time = "2026-08-25T23:32:04.429Z" }, + { url = "https://files.pythonhosted.org/packages/e6/eb/aa1f43ce2013a58e795f2dfe5ba92849e6b74ac93fcd2913dd847e0c1db0/liteparse-2.14.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:1896019aa736aac5f6594dd9433d1a15da5c9d868c75826ec97f470e987d33b7", size = 11971374, upload-time = "2026-08-25T23:32:06.841Z" }, + { url = "https://files.pythonhosted.org/packages/fb/20/727c2516e8626a8c83a6de46b2edbb7d7b2975edd5ddc39426ee11d82e6d/liteparse-2.14.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7219c9f9df82d46e1f29e56c7aa90f965c40067c92032d9d543f8b9c48b600ce", size = 13495316, upload-time = "2026-08-25T23:32:08.896Z" }, + { url = "https://files.pythonhosted.org/packages/d1/e0/c472f229287e4d4c1617ae07aeb2f8cc35f3bd99af7a7f8b17c87ff4a537/liteparse-2.14.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:91a6794b53425010a506cd8403a953c13e006039b593d4ab0cd36ad812e2cb9e", size = 13792828, upload-time = "2026-08-25T23:32:10.896Z" }, + { url = "https://files.pythonhosted.org/packages/e7/fd/7f0e19553301a56e24a413f5c2a0d8045bfd74ac2e870bf6b6aa58fc0da1/liteparse-2.14.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:90340ba3467df9f816b5cb866f7041b034d0da6de83559707795bd4edf5ea13d", size = 16968115, upload-time = "2026-08-25T23:32:12.938Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a9/f748a731a4b62f0d839ae03f4b008f81e5a1e2a210524648884b4114d59d/liteparse-2.14.0-cp310-abi3-win_amd64.whl", hash = "sha256:a864a5c939ca8061c9ecb02a7f80dde3cda452ec64394064f9514aee4b9de97a", size = 11996465, upload-time = "2026-08-25T23:32:15.114Z" }, + { url = "https://files.pythonhosted.org/packages/77/77/5995104db59e9d34d9158c27623ca92a8d6b43cf28fdf5aa0d3b4b86e556/liteparse-2.14.0-cp310-abi3-win_arm64.whl", hash = "sha256:28bcc45df09cacf9d75e946b8ccd21c48021912802835ec2375fdc16a185973b", size = 11306309, upload-time = "2026-08-25T23:32:17.563Z" }, +] + [[package]] name = "lxml" version = "6.0.2" @@ -4142,6 +4212,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/59/8c/b1c87148aa15e099243ec9f0cf9d0e970cc2234c3257d558c25a2c5304e6/tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f01a9c019878532f98927d2bacb79bbb404b43d3437455522a00a30718cdedb5", size = 3373542, upload-time = "2026-01-05T10:40:52.803Z" }, ] +[[package]] +name = "tokie" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/7e/ccb0f35fcdaea03ea4db7b4f6066aeec63d71bebf83d8181cda2fa320661/tokie-0.1.4.tar.gz", hash = "sha256:c8707df026d79a13228d410d607fde0dff5860d0f45ce11d89d85fb0a4c2a114", size = 280777, upload-time = "2026-07-24T16:41:15.502Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/f8/6507e5ef1fa3ccf8d8b453787ae5e8dd3214bdf925194442adad9b72e71c/tokie-0.1.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:80ff46843015605d2c3e34889f68a370d41d0f257e0edee4a4c253916b9eb3ba", size = 2866087, upload-time = "2026-07-24T16:40:35.314Z" }, + { url = "https://files.pythonhosted.org/packages/d0/3e/2b23f018a1bced6603a049c2c73b6bf902da17a5956863847b81b06c2133/tokie-0.1.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:299a87141ad6802fb48c7146dae3496c3716fc54f99e2a909b32c742780fa03a", size = 2897113, upload-time = "2026-07-24T16:40:37.475Z" }, + { url = "https://files.pythonhosted.org/packages/94/b0/c3bb8317d854c6da6a56f646bfd99ef7632c71b837462f06c8af15dbbebf/tokie-0.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:450e7270206afad356cad1e7d9c83cda98e5a0a8e633957df0abc484ddc08589", size = 3095044, upload-time = "2026-07-24T16:40:38.995Z" }, + { url = "https://files.pythonhosted.org/packages/3e/f5/839989186caa2769936f1e0a5dac7d33e6d73a46a958a164f8292cecc887/tokie-0.1.4-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:440c992d28f457f79ff244f1b8291fa0e245d058f131619dee728b99ba7e5eb8", size = 3207176, upload-time = "2026-07-24T16:40:40.588Z" }, + { url = "https://files.pythonhosted.org/packages/5b/0e/5c6cf9a817513dcc081c922a333b0a54c2d9ad506d32ba2d31a1f864e40d/tokie-0.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:0bc02cdd9bd9a15d997a7b5f0beb3e148d9c3efd5d949b9f51077f6c53aef6d1", size = 2639034, upload-time = "2026-07-24T16:40:42.367Z" }, + { url = "https://files.pythonhosted.org/packages/19/b6/53cd60a1182ce475812ae38ecaff15cfba4b447976920289f2caf471b14c/tokie-0.1.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4d922e3ba6635ecd513cfd6b8cb0969e88c0b329a41cbc0dd0c1a2c9053800f0", size = 2865741, upload-time = "2026-07-24T16:40:43.852Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b8/cf7a51c9d08bd2e5f4f8510c4da20b558bb8056e215a6045333308c07f69/tokie-0.1.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d06278ae88011f0fc6ff99d36707c856f633131f94dfc35f2033d0a71107d4bf", size = 2897068, upload-time = "2026-07-24T16:40:45.457Z" }, + { url = "https://files.pythonhosted.org/packages/b0/04/e447decd69987e3f4c00015ed4cd08106d851fb9ab2e9f34643d5cff7df4/tokie-0.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8a5b5dd143d807981711e973a9fff21a4fc48e472ef3a976054be04ac236a63", size = 3094709, upload-time = "2026-07-24T16:40:46.855Z" }, + { url = "https://files.pythonhosted.org/packages/de/d9/3b10e0c2c753db3b85a55d891548796f2fc16e5000941167ff2c6f3c6428/tokie-0.1.4-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d3ef8dcf454b1601d80ef8cfb5c5b86f89c8dd26f68dd05a5c25248872d20039", size = 3206948, upload-time = "2026-07-24T16:40:48.266Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/41aeae9d70bb7484ca4e4a9166d5330781b4a59b3cb8f1413a6b62e28204/tokie-0.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:885691c40fe47b1daaff1e7c9d13cc2f9734c8cd089a00fee5d05ee0f9a703d0", size = 2638905, upload-time = "2026-07-24T16:40:49.904Z" }, + { url = "https://files.pythonhosted.org/packages/7b/af/fae60b61803429d29f0ab5d01e474ff8461496d91c6c7753a7cede5d8db7/tokie-0.1.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3105e5d0df6829fddc0a6761e726dcae80cb8b373f39fe6396a32a2014157478", size = 2865113, upload-time = "2026-07-24T16:40:52.305Z" }, + { url = "https://files.pythonhosted.org/packages/4a/3f/ee12c8b77393ce6ceb5c465908b43ab96af4499fff88bff7e16f13022b09/tokie-0.1.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4ad390232d70e8e4b239e6e7edd8578e56433014ad496ea3597faa677b5e18b3", size = 2889620, upload-time = "2026-07-24T16:40:53.665Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f4/32eaa1aa37a5ad86244a1315096401cccefbb82c2c723b7181a42ee668e0/tokie-0.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eecd300fa19704fb3022aaedf8c1eafa9d9825c87ee13d2203e383bab4e764f", size = 3089894, upload-time = "2026-07-24T16:40:55.126Z" }, + { url = "https://files.pythonhosted.org/packages/62/66/b630a22d546da4712765e0472c4dbfdf50dcfc77b821ad274b12ee7be469/tokie-0.1.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:bbab9991a569beb667a0fe5f7c5698050914a46c015d0d323d8f4ef9e8c8aee2", size = 3202845, upload-time = "2026-07-24T16:40:56.571Z" }, + { url = "https://files.pythonhosted.org/packages/4c/64/e2bf4606adb89c942f72524b2a5a832cb6899676a623df04adaea1eee5b0/tokie-0.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:b0d245887bb806c42593304abce545fd1e7808c091946a9f3923d9211e99f4ff", size = 2635997, upload-time = "2026-07-24T16:40:58.283Z" }, + { url = "https://files.pythonhosted.org/packages/96/15/3fe8af5fe7ee607a6f6817e1119000b8bc828c9322dd68cccc724334ce38/tokie-0.1.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:e28c1c825479c8783ef2917bcc75964113d16ece254f9cd83234d19f3eaf7ff3", size = 2864983, upload-time = "2026-07-24T16:40:59.627Z" }, + { url = "https://files.pythonhosted.org/packages/ea/3f/35246c2ea85eb773342c64521da2481b0ff363cf1b39b1118eb6ab1c2ce3/tokie-0.1.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:199738e086f4a32892e651d9f98d10e606d12d581563041183e5c1b37a83745b", size = 2890250, upload-time = "2026-07-24T16:41:01.144Z" }, + { url = "https://files.pythonhosted.org/packages/f0/6c/114dfa287748d1bc32a0ff2ea56c24284a8afa4b9a1a391c65d2b20fed90/tokie-0.1.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2c41b8565e22aa0692e4cfcab595c0fe8b7ee742874e884ab9df928aa16d70e", size = 3089343, upload-time = "2026-07-24T16:41:02.629Z" }, + { url = "https://files.pythonhosted.org/packages/82/45/3e0fb358bd6fc3a9166cb7472f48055d7e894188b6f7553ebbad9e4e81f6/tokie-0.1.4-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:b39bf238b7b89bce19fa8ee353581ecffed46a7aac038502b9649789854b7e51", size = 3202698, upload-time = "2026-07-24T16:41:04.36Z" }, + { url = "https://files.pythonhosted.org/packages/54/99/d2444072b3ad7f2bcb7c40286e71d39f1cb968ebc52a90e6de31d5e5782d/tokie-0.1.4-cp313-cp313-win_amd64.whl", hash = "sha256:3ded2223b0486c8cf7922b9cf632ff10ee334bb7c736094f257515115826de87", size = 2635505, upload-time = "2026-07-24T16:41:05.533Z" }, + { url = "https://files.pythonhosted.org/packages/0f/32/4a0f5a30c5f31868fe795f6230cfad64545ed4900ee98f9f16f688f9dbb7/tokie-0.1.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:29013b3b7b6a6f3ed4b2e1fe35eeb4702236088c59bf137f8cb85954d69d5e84", size = 2866161, upload-time = "2026-07-24T16:41:06.973Z" }, + { url = "https://files.pythonhosted.org/packages/e2/db/150c266d98d5093fb65c9d150c8594d598a2b2626e6c4b88907552659d00/tokie-0.1.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b80085348b77d7e783513bfdc79dd4452c0602a28e04907124dcc17cbed8e5", size = 2890795, upload-time = "2026-07-24T16:41:08.397Z" }, + { url = "https://files.pythonhosted.org/packages/4c/42/078cc772347e8b6279fd9b643b5c7fb47eb49fe4eef88e3b8bbe165bbffc/tokie-0.1.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25344e4c02e47fc8cd2a146b1aa4ea5286b478bcbc5b109b1ec1da8d0ac17ff1", size = 3090930, upload-time = "2026-07-24T16:41:09.851Z" }, + { url = "https://files.pythonhosted.org/packages/6b/3f/8267161d70197f25ebe33e01907cac6a427228357b4ff9653900020f0e1a/tokie-0.1.4-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:cbc7b468fba5d5f506b85bb4319ffe825ff2c328aaea3c8ff57a36740af00e6d", size = 3203012, upload-time = "2026-07-24T16:41:11.375Z" }, + { url = "https://files.pythonhosted.org/packages/78/36/b97834b2152c24e6c5a5aaa39b6e4659105b3b3c472877e354d19baf3237/tokie-0.1.4-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:9b90d2ac37476497b7f5de25df4208b83d6a4a3cf43c790b7b91dcbc5419da8b", size = 958833, upload-time = "2026-07-24T16:41:12.715Z" }, + { url = "https://files.pythonhosted.org/packages/85/9d/d727a3a0d05bb78cc839e77a46b00bf8542769384ac6aa64c953314d5bfc/tokie-0.1.4-cp314-cp314-win_amd64.whl", hash = "sha256:2d4fb5a6b9cd4e137c3e6429129a5901909414552f35236c213668fbeab6cc77", size = 2637400, upload-time = "2026-07-24T16:41:14.288Z" }, +] + [[package]] name = "tomli" version = "2.4.0" From 99ec6a244647a1a65934cec96b4c38a44c7f3f15 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Wed, 26 Aug 2026 14:14:18 -0700 Subject: [PATCH 02/16] build(server): fetch the lance extension instead of INSTALL-ing it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A release builds linux/amd64,linux/arm64 on one amd64 runner via setup-qemu-action, so the non-native stage runs under qemu-user — where `import duckdb` segfaults before it can execute anything. `RUN python -c "import duckdb; ... INSTALL lance"` would therefore have failed every multi-arch release build. `install_lance_extension.py` reads the version from package metadata and downloads the extension for the stage's own architecture, never loading the native module. Verified both ways in one emulated x86_64 container: the script wrote the 254 MB linux_amd64 extension while `import duckdb` core-dumped beside it. The extension CDN 403s the default Python-urllib agent, hence the explicit one. `LOAD lance` in the smoke script is what proves the manual placement matches the path DuckDB resolves. --- extralit-server/docker/server/Dockerfile | 8 ++-- .../server/scripts/install_lance_extension.py | 46 +++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 extralit-server/docker/server/scripts/install_lance_extension.py diff --git a/extralit-server/docker/server/Dockerfile b/extralit-server/docker/server/Dockerfile index 47c965d7a..906951190 100644 --- a/extralit-server/docker/server/Dockerfile +++ b/extralit-server/docker/server/Dockerfile @@ -47,7 +47,7 @@ VOLUME $EXTRALIT_HOME_PATH # liteparse would otherwise download the language data on the first scanned page it meets. ENV TESSDATA_PREFIX=/usr/share/tesseract-ocr/5/tessdata -COPY scripts/start_extralit_server.sh scripts/smoke_retrieval_deps.py /home/extralit/ +COPY scripts/start_extralit_server.sh scripts/smoke_retrieval_deps.py scripts/install_lance_extension.py /home/extralit/ # Destination folder must be the same as the builder one. Otherwise installed script won't work (since the installation fixes the path inside the script) COPY --chown=extralit:extralit --from=builder /opt/venv /opt/venv @@ -60,9 +60,9 @@ RUN chmod +x start_extralit_server.sh USER extralit -# Pull the community `lance` extension now: DuckDB resolves it per user home, and a hybrid -# search must not be the thing that discovers the runtime has no network. -RUN python -c "import duckdb; duckdb.connect().execute('INSTALL lance')" +# DuckDB resolves extensions per user home, so this runs as `extralit`: a hybrid search must +# not be the thing that discovers the runtime has no network. +RUN python install_lance_extension.py # Exposing ports EXPOSE 6900 diff --git a/extralit-server/docker/server/scripts/install_lance_extension.py b/extralit-server/docker/server/scripts/install_lance_extension.py new file mode 100644 index 000000000..75e525fba --- /dev/null +++ b/extralit-server/docker/server/scripts/install_lance_extension.py @@ -0,0 +1,46 @@ +"""Bake DuckDB's `lance` extension into the image, for this stage's target architecture. + +`INSTALL lance` would be the obvious way to do this, but a release builds the non-native +arch under QEMU, where importing duckdb segfaults. Nothing here loads the native module: +the version comes from package metadata and the payload is a plain download, so the step +survives emulation. `smoke_retrieval_deps.py` is what proves the file landed where DuckDB +looks for it. +""" + +from __future__ import annotations + +import gzip +import platform +import urllib.request +from importlib.metadata import version +from pathlib import Path + +REPOSITORY = "http://extensions.duckdb.org" +# The CDN 403s the default Python-urllib agent; any identifiable one is served. +USER_AGENT = "extralit-server image build" +PLATFORMS = {"aarch64": "linux_arm64", "x86_64": "linux_amd64"} + + +def main() -> None: + machine = platform.machine() + try: + target = PLATFORMS[machine] + except KeyError: + raise SystemExit(f"no DuckDB extension platform for {machine}") from None + + duckdb_version = f"v{version('duckdb')}" + destination = Path.home() / ".duckdb" / "extensions" / duckdb_version / target + destination.mkdir(parents=True, exist_ok=True) + + url = f"{REPOSITORY}/{duckdb_version}/{target}/lance.duckdb_extension.gz" + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request) as response: + payload = gzip.decompress(response.read()) + + written = destination / "lance.duckdb_extension" + written.write_bytes(payload) + print(f"wrote {written} ({len(payload)} bytes)") + + +if __name__ == "__main__": + main() From b8b3948ec858d0f1bc107dba937564bf090385ef Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Wed, 26 Aug 2026 18:57:50 -0700 Subject: [PATCH 03/16] build(server): split the builder stage and apply the uv Docker guide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder was one RUN that copied the wheel, then ran apt-get update/upgrade/install, the install, and an apt-get purge. Because `COPY dist/*.whl` sat above it, a wheel change — the only input that differs between two builds of the same tree — invalidated the whole apt cycle. Measured on the same changed wheel: the old layout re-runs `python -m venv` and the full apt install/purge; the new one keeps both apt layers and the venv CACHED and re-runs only from the wheel COPY down. apt now stands alone at the top of the stage, and gcc/libc6-dev are simply left there rather than purged: the runtime image copies nothing out of the builder but /opt/venv, so there is no toolchain to remove (verified absent from the final image). From the uv Docker guide: UV_COMPILE_BYTECODE, UV_LINK_MODE=copy (the cache mount is a different filesystem from /opt/venv), UV_PYTHON_DOWNLOADS=0, VIRTUAL_ENV, and the cache mount moved to /root/.cache/uv to match UV_CACHE_DIR in a stage that has no extralit user. Bytecode compilation costs 150 MB (1.8 -> 1.95 GB) and halves cold start: importing extralit_server goes 23.0s -> 12.5s, paid back on every worker and every Space wake. `uv venv --seed`, not `uv venv`: extralit-hf-space derives from this image and installs into this venv with `pip`, which a default uv venv does not create. --- extralit-server/docker/server/Dockerfile | 41 +++++++++++++++--------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/extralit-server/docker/server/Dockerfile b/extralit-server/docker/server/Dockerfile index 906951190..d4ecb5e1f 100644 --- a/extralit-server/docker/server/Dockerfile +++ b/extralit-server/docker/server/Dockerfile @@ -1,23 +1,34 @@ FROM python:3.12-slim AS builder -# Install uv COPY --from=ghcr.io/astral-sh/uv:0.7.12 /uv /uvx /bin/ -# Copying extralit distribution files -COPY dist/*.whl /packages/ -RUN python -m venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" -ENV MAMBA_ROOT_PREFIX=/opt/venv -ENV CONDA_PREFIX=/opt/venv -ENV UV_CACHE_DIR=/home/extralit/.cache/uv -RUN --mount=type=cache,target=/home/extralit/.cache/uv \ - apt-get update && \ +ENV VIRTUAL_ENV=/opt/venv \ + PATH="/opt/venv/bin:$PATH" \ + MAMBA_ROOT_PREFIX=/opt/venv \ + CONDA_PREFIX=/opt/venv \ + UV_CACHE_DIR=/root/.cache/uv \ + # Pay for the .pyc files once here instead of on every worker's first import. + UV_COMPILE_BYTECODE=1 \ + # The cache mount is a different filesystem from /opt/venv, so hardlinking cannot work. + UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=0 + +# psycopg2 is built from source. The toolchain stays in this stage — the runtime image copies +# nothing out of it but the venv, so there is nothing to purge afterwards. +RUN apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends libc6-dev libpq-dev gcc && \ - for wheel in /packages/*.whl; do uv pip install "$wheel"[postgresql]; done && \ - apt-get purge -y --auto-remove libc6-dev libpq-dev gcc && \ apt-get clean && \ - rm -rf /var/lib/apt/lists/* /packages + rm -rf /var/lib/apt/lists/* + +# --seed: the HF-Space image derives from this one and installs into this venv with `pip`. +RUN uv venv --seed "$VIRTUAL_ENV" + +# Copied last: the wheel is the only input that differs between two builds of the same tree, +# so everything above it survives in the layer cache. +COPY dist/*.whl /packages/ +RUN --mount=type=cache,target=/root/.cache/uv \ + for wheel in /packages/*.whl; do uv pip install "$wheel"[postgresql]; done FROM python:3.12-slim @@ -47,16 +58,16 @@ VOLUME $EXTRALIT_HOME_PATH # liteparse would otherwise download the language data on the first scanned page it meets. ENV TESSDATA_PREFIX=/usr/share/tesseract-ocr/5/tessdata -COPY scripts/start_extralit_server.sh scripts/smoke_retrieval_deps.py scripts/install_lance_extension.py /home/extralit/ +COPY --chmod=0755 scripts/start_extralit_server.sh scripts/smoke_retrieval_deps.py scripts/install_lance_extension.py /home/extralit/ # Destination folder must be the same as the builder one. Otherwise installed script won't work (since the installation fixes the path inside the script) COPY --chown=extralit:extralit --from=builder /opt/venv /opt/venv +ENV VIRTUAL_ENV=/opt/venv ENV PATH="/opt/venv/bin:$PATH" ENV MAMBA_ROOT_PREFIX=/opt/venv ENV CONDA_PREFIX=/opt/venv WORKDIR /home/extralit -RUN chmod +x start_extralit_server.sh USER extralit From a25b574318eb1b01c28518d395fcb941479235a1 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Wed, 26 Aug 2026 19:04:24 -0700 Subject: [PATCH 04/16] feat(ocr): read `items` rows back as typed elements, with a liftable table header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First slice of Phase 1. `elements_from_items` is the inverse of `arrow.item_rows`: it reads the Lance rows back as the three units a chunker dispatches on — markdown, table, figure — so chunking can re-run from the dataset without re-parsing the PDF, and `contexts/retrieval` never has to know a docling label. Decisions the rows forced: - One element per provenance row, not per item, so an item spanning a page break stays two elements with two bboxes instead of one claiming to be in two places. Text is sliced by charspan when an item has several provenances. - Captions are consumed by the nearest figure or table on the same page, either side of it: nothing here links a PictureItem to its caption, and geometric parsers order them both ways. A caption with no figure on its page survives as prose rather than being dropped. - An uncaptioned figure produces no element. There is nothing retrievable in it. - page_header/page_footer are dropped; running furniture is repeated on every page and retrievable on none. - Headings render as ATX markdown so the recursive chunker's line-anchored rules can split on them, and each element carries its breadcrumb. A title holds a slot above every section header whatever its level, so `Results` closes `Methods` without closing the title. `table_html` replaces docling's exporter, which puts the header's `` cells inside `` — a header a row-window chunk cannot find is a header it cannot repeat. The layout API's `html` field changes shape with it; no frontend reads it and the OpenAPI type is unchanged. --- .../src/extralit_server/contexts/ocr/arrow.py | 50 +++- .../extralit_server/contexts/ocr/elements.py | 178 ++++++++++++ .../tests/unit/contexts/ocr/test_elements.py | 258 ++++++++++++++++++ 3 files changed, 484 insertions(+), 2 deletions(-) create mode 100644 extralit-server/src/extralit_server/contexts/ocr/elements.py create mode 100644 extralit-server/tests/unit/contexts/ocr/test_elements.py diff --git a/extralit-server/src/extralit_server/contexts/ocr/arrow.py b/extralit-server/src/extralit_server/contexts/ocr/arrow.py index 60136d74a..501f081eb 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/arrow.py +++ b/extralit-server/src/extralit_server/contexts/ocr/arrow.py @@ -7,10 +7,12 @@ from __future__ import annotations +import html +from collections import defaultdict from typing import Any, Optional import pyarrow as pa -from docling_core.types.doc import DoclingDocument +from docling_core.types.doc import DoclingDocument, TableCell from docling_core.types.doc.document import DocItem, TableItem ITEM_SCHEMA = pa.schema( @@ -49,11 +51,55 @@ def _enum_value(value: Any) -> Optional[str]: return getattr(value, "value", value) +def _cell_html(cell: TableCell, in_header: bool) -> str: + tag = "th" if in_header or cell.column_header or cell.row_header else "td" + spans = "" + if cell.col_span > 1: + spans += f' colspan="{cell.col_span}"' + if cell.row_span > 1: + spans += f' rowspan="{cell.row_span}"' + return f"<{tag}{spans}>{html.escape(cell.text or '', quote=False)}" + + +def table_html(item: TableItem) -> Optional[str]: + """Serialize a table with a real ``. + + docling's own exporter puts the header's `` cells inside ``, which leaves the + header unliftable — and a table split into row windows has to repeat it on every chunk. + """ + cells = list(getattr(item.data, "table_cells", None) or []) + if not cells: + return None + + by_row: dict[int, list[TableCell]] = defaultdict(list) + for cell in cells: + by_row[cell.start_row_offset_idx].append(cell) + rows = [sorted(by_row[index], key=lambda c: c.start_col_offset_idx) for index in sorted(by_row)] + + # Only a leading run of all-header rows is a header; a stray `column_header` further down + # is a mislabelled body cell, not the start of a second one. + header_rows = 0 + for row in rows: + if not all(cell.column_header for cell in row): + break + header_rows += 1 + + def section(tag: str, group: list[list[TableCell]], in_header: bool) -> str: + if not group: + return "" + body = "".join("" + "".join(_cell_html(c, in_header) for c in row) + "" for row in group) + return f"<{tag}>{body}" + + head = section("thead", rows[:header_rows], True) + body = section("tbody", rows[header_rows:], False) + return f"{head}{body}
" + + def _table_html(doc: DoclingDocument, item: DocItem) -> Optional[str]: if not isinstance(item, TableItem): return None try: - return item.export_to_html(doc=doc) or None + return table_html(item) except Exception: # a malformed table must not sink the whole projection return None diff --git a/extralit-server/src/extralit_server/contexts/ocr/elements.py b/extralit-server/src/extralit_server/contexts/ocr/elements.py new file mode 100644 index 000000000..ae0540951 --- /dev/null +++ b/extralit-server/src/extralit_server/contexts/ocr/elements.py @@ -0,0 +1,178 @@ +"""Typed view of the layout store's `items` rows, one step below chunking. + +`arrow.item_rows` flattens a `DoclingDocument` into columns; this reads those columns back as +the three units a chunker dispatches on — a markdown run, a table, or a figure with its caption +— so `contexts/retrieval` never has to know a docling label. Rows are the only input, which is +what lets chunking re-run from the Lance dataset without re-parsing the PDF. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from typing import Optional + +from docling_core.types.doc import DocItemLabel + +from extralit_server.contexts.ocr.docling_builder import PICTURE_LABELS, TABLE_LABELS + +MARKDOWN = "markdown" +TABLE = "table" +FIGURE = "figure" + +#: Running page furniture: repeated on every page, retrievable on none of them. +SKIPPED_LABELS = frozenset({DocItemLabel.PAGE_HEADER, DocItemLabel.PAGE_FOOTER}) + +#: Headings open a breadcrumb slot. A title sits above every section header, whatever its level. +TITLE_SLOT = 0 + +_MAX_HEADING_LEVEL = 6 + + +@dataclass(frozen=True) +class Element: + """One retrievable unit, carrying the provenance a hit needs to point back at the page.""" + + type: str + content: str + page_no: Optional[int] + bbox: Optional[tuple[float, float, float, float]] + label: str + level: Optional[int] + item_ref: str + reading_order: int + headings: tuple[str, ...] = () + + +def _text_of(row: Mapping, siblings: int) -> str: + """A row's own text, sliced to its provenance when one item spans several of them.""" + text = row.get("text") or "" + if siblings < 2: + return text + start, end = row.get("charspan_start"), row.get("charspan_end") + if start is None or end is None or end <= start: + return text + return text[start:end] + + +def _markdown(label: str, text: str, level: Optional[int]) -> str: + """Render a row as the markdown the recursive chunker's rules are written against.""" + if label == DocItemLabel.TITLE: + return f"# {text}" + if label == DocItemLabel.SECTION_HEADER: + return "#" * min(max(level or 1, 1), _MAX_HEADING_LEVEL) + f" {text}" + if label == DocItemLabel.LIST_ITEM: + return f"- {text}" + if label == DocItemLabel.CODE: + return f"```\n{text}\n```" + return text + + +def _heading_slot(label: str, level: Optional[int]) -> Optional[int]: + if label == DocItemLabel.TITLE: + return TITLE_SLOT + if label == DocItemLabel.SECTION_HEADER: + return max(level or 1, 1) + return None + + +def _push_heading(stack: list[tuple[int, str]], slot: int, text: str) -> None: + """Open a breadcrumb slot, closing every slot at or below it.""" + while stack and stack[-1][0] >= slot: + stack.pop() + stack.append((slot, text)) + + +def _bbox_of(row: Mapping) -> Optional[tuple[float, float, float, float]]: + bbox = row.get("bbox") + return tuple(float(v) for v in bbox) if bbox else None # ty: ignore[invalid-return-type] + + +def _captionable(rows: Sequence[Mapping], index: int) -> Optional[int]: + """Index of the figure or table a caption at `index` belongs to. + + docling puts a caption after its figure, but parsers that sort geometrically can put it + either side, so the nearer neighbour on the same page wins. + """ + page = rows[index].get("page_no") + best: Optional[tuple[int, int]] = None + for offset in (-1, 1, -2, 2): + neighbour = index + offset + if not 0 <= neighbour < len(rows): + continue + row = rows[neighbour] + if row.get("page_no") != page: + continue + if row.get("label") in TABLE_LABELS or row.get("label") in PICTURE_LABELS: + distance = abs(offset) + if best is None or distance < best[0]: + best = (distance, neighbour) + return best[1] if best else None + + +def elements_from_items(rows: Iterable[Mapping]) -> list[Element]: + """Read `items` rows back into elements, in reading order. + + One element per row, so an item spanning a page break stays two elements with two bboxes + rather than one element that claims to be in two places. + """ + ordered = sorted(rows, key=lambda r: (r.get("reading_order") or 0, r.get("prov_index") or 0)) + provs: dict[str, int] = {} + for row in ordered: + provs[row.get("self_ref")] = provs.get(row.get("self_ref"), 0) + 1 + + # Captions are consumed by the figure or table they describe, so resolve them first. + captions: dict[int, str] = {} + consumed: set[int] = set() + for index, row in enumerate(ordered): + if row.get("label") != DocItemLabel.CAPTION: + continue + owner = _captionable(ordered, index) + if owner is None: + continue + text = _text_of(row, provs.get(row.get("self_ref"), 1)).strip() + if not text: + continue + captions[owner] = f"{captions[owner]} {text}" if owner in captions else text + consumed.add(index) + + elements: list[Element] = [] + stack: list[tuple[int, str]] = [] + + for index, row in enumerate(ordered): + label = row.get("label") or DocItemLabel.TEXT + if label in SKIPPED_LABELS or index in consumed: + continue + + level = row.get("level") + text = _text_of(row, provs.get(row.get("self_ref"), 1)).strip() + + slot = _heading_slot(label, level) + if slot is not None and text: + _push_heading(stack, slot, text) + + if label in TABLE_LABELS: + kind, content = TABLE, (row.get("html") or "") + elif label in PICTURE_LABELS: + kind, content = FIGURE, captions.get(index, "") + else: + kind, content = MARKDOWN, _markdown(label, text, level) if text else "" + + if not content: + continue + + elements.append( + Element( + type=kind, + content=content, + page_no=row.get("page_no"), + bbox=_bbox_of(row), + label=str(label), + level=level, + item_ref=row.get("self_ref"), + reading_order=row.get("reading_order") or 0, + headings=tuple(heading for _, heading in stack), + ) + ) + + return elements diff --git a/extralit-server/tests/unit/contexts/ocr/test_elements.py b/extralit-server/tests/unit/contexts/ocr/test_elements.py new file mode 100644 index 000000000..926d0004c --- /dev/null +++ b/extralit-server/tests/unit/contexts/ocr/test_elements.py @@ -0,0 +1,258 @@ +"""Tests for the typed element view of `items` rows.""" + +import pytest +from docling_core.types.doc import BoundingBox, CoordOrigin, DocItemLabel, Size + +from extralit_server.contexts.ocr.arrow import item_rows, table_html +from extralit_server.contexts.ocr.docling_builder import ( + LayoutBlock, + PageContext, + append_blocks, + new_document, +) +from extralit_server.contexts.ocr.elements import FIGURE, MARKDOWN, TABLE, elements_from_items +from extralit_server.contexts.ocr.tables import make_cell + +DOCUMENT_ID = "11111111-2222-3333-4444-555555555555" + + +def bbox(t: float, b: float, left: float = 10.0, right: float = 100.0) -> BoundingBox: + return BoundingBox(l=left, t=t, r=right, b=b, coord_origin=CoordOrigin.TOPLEFT) + + +def row(label, reading_order, **overrides): + base = { + "self_ref": f"#/texts/{reading_order}", + "label": label, + "level": None, + "reading_order": reading_order, + "prov_index": 0, + "page_no": 1, + "bbox": [0.0, 0.0, 10.0, 10.0], + "text": None, + "html": None, + "charspan_start": None, + "charspan_end": None, + } + base.update(overrides) + return base + + +class TestHeadingBreadcrumb: + def test_breadcrumb_deepens_and_unwinds_with_heading_level(self): + rows = [ + row(DocItemLabel.TITLE, 0, text="A Paper"), + row(DocItemLabel.SECTION_HEADER, 1, level=1, text="Methods"), + row(DocItemLabel.SECTION_HEADER, 2, level=2, text="Sampling"), + row(DocItemLabel.TEXT, 3, text="Deep body."), + row(DocItemLabel.SECTION_HEADER, 4, level=1, text="Results"), + row(DocItemLabel.TEXT, 5, text="Shallow body."), + ] + + elements = elements_from_items(rows) + + assert [e.headings for e in elements] == [ + ("A Paper",), + ("A Paper", "Methods"), + ("A Paper", "Methods", "Sampling"), + ("A Paper", "Methods", "Sampling"), + # Results closes Methods and Sampling but never the title. + ("A Paper", "Results"), + ("A Paper", "Results"), + ] + + def test_a_heading_carries_itself_so_a_chunk_knows_its_own_path(self): + rows = [row(DocItemLabel.SECTION_HEADER, 0, level=1, text="Methods")] + + assert elements_from_items(rows)[0].headings == ("Methods",) + + +class TestMarkdownRendering: + @pytest.mark.parametrize( + "label, level, text, expected", + [ + (DocItemLabel.TITLE, None, "A Paper", "# A Paper"), + (DocItemLabel.SECTION_HEADER, 3, "Deep", "### Deep"), + (DocItemLabel.SECTION_HEADER, None, "Unlevelled", "# Unlevelled"), + (DocItemLabel.LIST_ITEM, None, "first", "- first"), + (DocItemLabel.TEXT, None, "Prose.", "Prose."), + ], + ) + def test_rows_render_as_the_markdown_the_chunker_rules_expect(self, label, level, text, expected): + elements = elements_from_items([row(label, 0, level=level, text=text)]) + + assert elements[0].type == MARKDOWN + assert elements[0].content == expected + + def test_heading_level_is_clamped_to_six(self): + elements = elements_from_items([row(DocItemLabel.SECTION_HEADER, 0, level=99, text="Deep")]) + + assert elements[0].content == "###### Deep" + + +class TestCaptions: + def test_a_caption_is_absorbed_by_its_figure_rather_than_left_as_prose(self): + rows = [ + row(DocItemLabel.PICTURE, 0), + row(DocItemLabel.CAPTION, 1, text="Figure 1. A red square."), + ] + + elements = elements_from_items(rows) + + assert [(e.type, e.content) for e in elements] == [(FIGURE, "Figure 1. A red square.")] + + def test_a_caption_preceding_its_figure_is_still_absorbed(self): + rows = [ + row(DocItemLabel.CAPTION, 0, text="Table 1. Counts."), + row(DocItemLabel.TABLE, 1, html="
x
"), + ] + + elements = elements_from_items(rows) + + assert [e.type for e in elements] == [TABLE] + + def test_a_caption_on_a_page_with_no_figure_survives_as_prose(self): + rows = [ + row(DocItemLabel.PICTURE, 0, page_no=2), + row(DocItemLabel.CAPTION, 1, page_no=1, text="Orphaned."), + ] + + elements = elements_from_items(rows) + + assert [(e.type, e.content) for e in elements] == [(MARKDOWN, "Orphaned.")] + + def test_an_uncaptioned_figure_yields_nothing_retrievable(self): + assert elements_from_items([row(DocItemLabel.PICTURE, 0)]) == [] + + +class TestProvenance: + def test_running_headers_and_footers_are_dropped(self): + rows = [ + row(DocItemLabel.PAGE_HEADER, 0, text="Journal of Things"), + row(DocItemLabel.TEXT, 1, text="Real body."), + row(DocItemLabel.PAGE_FOOTER, 2, text="7"), + ] + + assert [e.content for e in elements_from_items(rows)] == ["Real body."] + + def test_one_element_per_provenance_row_when_an_item_spans_a_page_break(self): + spanning = [ + row(DocItemLabel.TEXT, 0, page_no=1, text="first half second half", charspan_start=0, charspan_end=11), + row( + DocItemLabel.TEXT, + 0, + page_no=2, + prov_index=1, + text="first half second half", + charspan_start=11, + charspan_end=22, + ), + ] + + elements = elements_from_items(spanning) + + assert [(e.page_no, e.content) for e in elements] == [(1, "first half"), (2, "second half")] + + def test_elements_come_back_in_reading_order_whatever_the_row_order(self): + rows = [row(DocItemLabel.TEXT, 2, text="third"), row(DocItemLabel.TEXT, 0, text="first")] + + assert [e.content for e in elements_from_items(rows)] == ["first", "third"] + + def test_bbox_and_item_ref_survive_the_round_trip(self): + elements = elements_from_items([row(DocItemLabel.TEXT, 0, text="x", bbox=[1.0, 2.0, 3.0, 4.0])]) + + assert elements[0].bbox == (1.0, 2.0, 3.0, 4.0) + assert elements[0].item_ref == "#/texts/0" + + +class TestTableHtml: + @pytest.fixture + def table(self): + document = new_document("sample") + append_blocks( + document, + PageContext(page_no=1, size=Size(width=612, height=792)), + [ + LayoutBlock( + label=DocItemLabel.TABLE, + bbox=bbox(t=200, b=400), + cells=[ + make_cell("Group", row=0, col=0, column_header=True), + make_cell("N", row=0, col=1, column_header=True), + make_cell("control", row=1, col=0), + make_cell("4 < 2", row=1, col=1), + ], + ) + ], + ) + return document.tables[0] + + def test_header_cells_land_in_thead_not_tbody(self, table): + assert table_html(table) == ( + "" + "" + "" + "
GroupN
control4 < 2
" + ) + + def test_docling_own_exporter_would_have_buried_the_header(self, table): + # The reason this serializer exists: a row-window chunk cannot repeat a header it + # cannot find, and docling puts the cells inside . + assert "" not in table.export_to_html() + + def test_spans_are_carried_as_attributes(self): + document = new_document("sample") + append_blocks( + document, + PageContext(page_no=1, size=Size(width=612, height=792)), + [ + LayoutBlock( + label=DocItemLabel.TABLE, + bbox=bbox(t=200, b=400), + cells=[ + make_cell("Both", row=0, col=0, col_span=2, column_header=True), + make_cell("tall", row=1, col=0, row_span=2), + make_cell("x", row=1, col=1), + ], + ) + ], + ) + + html = table_html(document.tables[0]) + + assert 'Both' in html + assert 'tall' in html + + def test_a_table_with_no_cells_has_no_html(self): + document = new_document("sample") + append_blocks( + document, + PageContext(page_no=1, size=Size(width=612, height=792)), + [LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=200, b=400), cells=[])], + ) + + assert table_html(document.tables[0]) is None + + +class TestAgainstRealProjection: + def test_elements_line_up_with_a_projected_document(self): + document = new_document("sample") + append_blocks( + document, + PageContext(page_no=1, size=Size(width=612, height=792)), + [ + LayoutBlock(label=DocItemLabel.SECTION_HEADER, bbox=bbox(t=10, b=40), text="Methods", level=1), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=50, b=70), text="Body text."), + LayoutBlock( + label=DocItemLabel.TABLE, + bbox=bbox(t=200, b=400), + cells=[make_cell("N", row=0, col=0, column_header=True), make_cell("42", row=1, col=0)], + ), + ], + ) + + elements = elements_from_items(item_rows(document, DOCUMENT_ID)) + + assert [e.type for e in elements] == [MARKDOWN, MARKDOWN, TABLE] + assert all(e.headings == ("Methods",) for e in elements) + assert "" in elements[-1].content From 9db0f0f0b95301e4e4bbaaa4426fa158b03d1ebd Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Wed, 26 Aug 2026 21:52:00 -0700 Subject: [PATCH 05/16] build(server): pull the lance fetch out of the per-build path, bump uv, drop the conda vars The 231 MB extension was fetched in the final stage, below the venv COPY, so every one-line code change rebuilt and re-exported it. It now has its own stage above everything that varies per build: a wheel change leaves both the fetch and its COPY CACHED, and BuildKit runs the fetch in parallel with the wheel install. Incremental rebuild 30s -> 27s locally, and the big layer stops churning through the registry on CI. That needs the duckdb version before the venv exists, so DUCKDB_VERSION is an ARG. The pin is kept honest by `--check`, which compares it against the venv's resolved duckdb from package metadata (no native import, so it survives the emulated arch of a release build) and fails the build rather than shipping an image whose first hybrid search cannot find the extension. Verified by building with a deliberately wrong pin. Also: uv 0.7.12 -> 0.12.6; MAMBA_ROOT_PREFIX and CONDA_PREFIX dropped, vestiges of a micromamba base that nothing in either repo reads; the wheel is bind-mounted rather than copied, so no copy of it is left in the builder; and a .dockerignore allowlists the context down to the wheel and two scripts. --- extralit-server/docker/server/.dockerignore | 6 +++ extralit-server/docker/server/Dockerfile | 33 ++++++++---- .../server/scripts/install_lance_extension.py | 53 +++++++++++++++---- 3 files changed, 71 insertions(+), 21 deletions(-) create mode 100644 extralit-server/docker/server/.dockerignore diff --git a/extralit-server/docker/server/.dockerignore b/extralit-server/docker/server/.dockerignore new file mode 100644 index 000000000..ee0676668 --- /dev/null +++ b/extralit-server/docker/server/.dockerignore @@ -0,0 +1,6 @@ +# Allowlist: the context is a wheel and two scripts, and should stay that way however much +# stray build output lands in this directory. +* +!scripts/*.py +!scripts/*.sh +!dist/*.whl diff --git a/extralit-server/docker/server/Dockerfile b/extralit-server/docker/server/Dockerfile index d4ecb5e1f..3cd2a96b5 100644 --- a/extralit-server/docker/server/Dockerfile +++ b/extralit-server/docker/server/Dockerfile @@ -1,11 +1,20 @@ +# The duckdb the venv resolves. Pinned so the 231 MB extension fetch can sit in a stage that does +# not depend on the application wheel; the `--check` below fails the build if the pin ever drifts. +ARG DUCKDB_VERSION=1.5.5 + +# Fetched in its own stage, ahead of anything that changes per build: DuckDB looks extensions up +# under a version-named directory, so this layer is identical until duckdb itself moves. +FROM python:3.12-slim AS lance +ARG DUCKDB_VERSION +COPY scripts/install_lance_extension.py /tmp/ +RUN python /tmp/install_lance_extension.py --duckdb-home /lance --duckdb-version "$DUCKDB_VERSION" + FROM python:3.12-slim AS builder -COPY --from=ghcr.io/astral-sh/uv:0.7.12 /uv /uvx /bin/ +COPY --from=ghcr.io/astral-sh/uv:0.12.6 /uv /uvx /bin/ ENV VIRTUAL_ENV=/opt/venv \ PATH="/opt/venv/bin:$PATH" \ - MAMBA_ROOT_PREFIX=/opt/venv \ - CONDA_PREFIX=/opt/venv \ UV_CACHE_DIR=/root/.cache/uv \ # Pay for the .pyc files once here instead of on every worker's first import. UV_COMPILE_BYTECODE=1 \ @@ -24,13 +33,14 @@ RUN apt-get update && \ # --seed: the HF-Space image derives from this one and installs into this venv with `pip`. RUN uv venv --seed "$VIRTUAL_ENV" -# Copied last: the wheel is the only input that differs between two builds of the same tree, -# so everything above it survives in the layer cache. -COPY dist/*.whl /packages/ +# The wheel is the only input that differs between two builds of the same tree, so it enters +# last — and as a bind mount, which leaves no copy of it behind in the stage. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=dist,target=/packages \ for wheel in /packages/*.whl; do uv pip install "$wheel"[postgresql]; done FROM python:3.12-slim +ARG DUCKDB_VERSION # Environment Variables ENV USERNAME="" @@ -58,22 +68,23 @@ VOLUME $EXTRALIT_HOME_PATH # liteparse would otherwise download the language data on the first scanned page it meets. ENV TESSDATA_PREFIX=/usr/share/tesseract-ocr/5/tessdata +# Layers below are ordered by how often they change, because a rebuilt layer rebuilds every layer +# under it. DuckDB resolves extensions per user home, hence extralit's rather than root's. +COPY --from=lance --chown=extralit:extralit /lance /home/extralit/.duckdb COPY --chmod=0755 scripts/start_extralit_server.sh scripts/smoke_retrieval_deps.py scripts/install_lance_extension.py /home/extralit/ # Destination folder must be the same as the builder one. Otherwise installed script won't work (since the installation fixes the path inside the script) COPY --chown=extralit:extralit --from=builder /opt/venv /opt/venv ENV VIRTUAL_ENV=/opt/venv ENV PATH="/opt/venv/bin:$PATH" -ENV MAMBA_ROOT_PREFIX=/opt/venv -ENV CONDA_PREFIX=/opt/venv WORKDIR /home/extralit USER extralit -# DuckDB resolves extensions per user home, so this runs as `extralit`: a hybrid search must -# not be the thing that discovers the runtime has no network. -RUN python install_lance_extension.py +# Fails the build rather than shipping an image whose first hybrid search cannot load the +# extension. Reads metadata only, so it survives the emulated arch of a release build. +RUN python install_lance_extension.py --check --duckdb-version "$DUCKDB_VERSION" # Exposing ports EXPOSE 6900 diff --git a/extralit-server/docker/server/scripts/install_lance_extension.py b/extralit-server/docker/server/scripts/install_lance_extension.py index 75e525fba..aa9997094 100644 --- a/extralit-server/docker/server/scripts/install_lance_extension.py +++ b/extralit-server/docker/server/scripts/install_lance_extension.py @@ -1,14 +1,20 @@ """Bake DuckDB's `lance` extension into the image, for this stage's target architecture. -`INSTALL lance` would be the obvious way to do this, but a release builds the non-native -arch under QEMU, where importing duckdb segfaults. Nothing here loads the native module: -the version comes from package metadata and the payload is a plain download, so the step -survives emulation. `smoke_retrieval_deps.py` is what proves the file landed where DuckDB -looks for it. + python install_lance_extension.py --duckdb-home /lance --duckdb-version 1.5.5 + python install_lance_extension.py --check --duckdb-version 1.5.5 + +`INSTALL lance` would be the obvious way to fetch it, but a release builds the non-native arch +under QEMU, where importing duckdb segfaults. Nothing here loads the native module: the payload +is a plain download and `--check` reads package metadata, so both survive emulation. + +The version is passed in rather than derived so the fetch can live in a stage that does not +depend on the application wheel — otherwise a one-line code change re-downloads 231 MB. `--check` +is what keeps that pin honest: it fails the build if the venv resolved a different duckdb. """ from __future__ import annotations +import argparse import gzip import platform import urllib.request @@ -21,18 +27,30 @@ PLATFORMS = {"aarch64": "linux_arm64", "x86_64": "linux_amd64"} -def main() -> None: +def target_platform() -> str: machine = platform.machine() try: - target = PLATFORMS[machine] + return PLATFORMS[machine] except KeyError: raise SystemExit(f"no DuckDB extension platform for {machine}") from None - duckdb_version = f"v{version('duckdb')}" - destination = Path.home() / ".duckdb" / "extensions" / duckdb_version / target + +def check(expected: str) -> None: + installed = version("duckdb") + if installed != expected: + raise SystemExit( + f"the image bakes the lance extension for duckdb {expected}, but the venv resolved " + f"{installed}. DuckDB looks extensions up under a version-named directory, so this " + f"would ship an image whose first hybrid search fails. Set DUCKDB_VERSION={installed} " + f"in the Dockerfile." + ) + + +def fetch(duckdb_version: str, duckdb_home: Path) -> None: + destination = duckdb_home / "extensions" / f"v{duckdb_version}" / target_platform() destination.mkdir(parents=True, exist_ok=True) - url = f"{REPOSITORY}/{duckdb_version}/{target}/lance.duckdb_extension.gz" + url = f"{REPOSITORY}/v{duckdb_version}/{target_platform()}/lance.duckdb_extension.gz" request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) with urllib.request.urlopen(request) as response: payload = gzip.decompress(response.read()) @@ -42,5 +60,20 @@ def main() -> None: print(f"wrote {written} ({len(payload)} bytes)") +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--duckdb-version", help="defaults to the installed duckdb") + parser.add_argument("--duckdb-home", type=Path, default=Path.home() / ".duckdb") + parser.add_argument("--check", action="store_true", help="assert the venv agrees with the pin") + args = parser.parse_args() + + duckdb_version = args.duckdb_version or version("duckdb") + if args.check: + check(duckdb_version) + print(f"ok: venv duckdb matches the baked lance extension ({duckdb_version})") + return + fetch(duckdb_version, args.duckdb_home) + + if __name__ == "__main__": main() From 6c23323f9fa54af59080fa591f8db60aa792f252 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Wed, 26 Aug 2026 23:29:12 -0700 Subject: [PATCH 06/16] fix(server): verify the lance extension over TLS against a pinned digest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Roborev, Critical, jobs #379 and #382. The build fetched a native DuckDB extension over plaintext HTTP with no integrity check, and the server later loads that extension into its own process — so anything able to answer for extensions.duckdb.org could put native code inside the server. The fetch is now HTTPS, and the decompressed payload's sha256 is checked against a pin kept per duckdb version and target platform before a single byte is written. The two pinned digests were confirmed against the extension already baked into a built image, not just against a fresh download of themselves. It fails closed in both directions: a duckdb version with no pinned digest aborts the build rather than trusting whatever the repository serves, and a mismatch refuses to write. Verified all three paths — happy path, unpinned version, doctored digest (zero files written). Bumping DUCKDB_VERSION now also means adding a digest; `--digest` prints what the repository is currently serving, to be confirmed independently before pinning. --- .../server/scripts/install_lance_extension.py | 61 ++++++++++++++++--- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/extralit-server/docker/server/scripts/install_lance_extension.py b/extralit-server/docker/server/scripts/install_lance_extension.py index aa9997094..65b02a5bc 100644 --- a/extralit-server/docker/server/scripts/install_lance_extension.py +++ b/extralit-server/docker/server/scripts/install_lance_extension.py @@ -10,22 +10,36 @@ The version is passed in rather than derived so the fetch can live in a stage that does not depend on the application wheel — otherwise a one-line code change re-downloads 231 MB. `--check` is what keeps that pin honest: it fails the build if the venv resolved a different duckdb. + +This extension is native code that the server later loads into its own process, so the payload +is fetched over TLS and checked against a pinned digest before it is written. An unpinned +version fails the build rather than trusting whatever the network returned. """ from __future__ import annotations import argparse import gzip +import hashlib import platform import urllib.request from importlib.metadata import version from pathlib import Path -REPOSITORY = "http://extensions.duckdb.org" +REPOSITORY = "https://extensions.duckdb.org" # The CDN 403s the default Python-urllib agent; any identifiable one is served. USER_AGENT = "extralit-server image build" PLATFORMS = {"aarch64": "linux_arm64", "x86_64": "linux_amd64"} +#: sha256 of the *decompressed* extension, per duckdb version and target platform. Bumping +#: DUCKDB_VERSION means adding an entry here; see `--digest` for how to produce one. +DIGESTS = { + "1.5.5": { + "linux_amd64": "a8b1463e8541a960859b05c39096a60a3c777d12fd63d9867ce62bb251ab2a1a", + "linux_arm64": "a710b8c2453e996ff810c718ccc9b26253a1cf6c1b6687fad0dd805f8878e2c9", + }, +} + def target_platform() -> str: machine = platform.machine() @@ -35,6 +49,25 @@ def target_platform() -> str: raise SystemExit(f"no DuckDB extension platform for {machine}") from None +def expected_digest(duckdb_version: str, target: str) -> str: + try: + return DIGESTS[duckdb_version][target] + except KeyError: + raise SystemExit( + f"no pinned sha256 for the lance extension at duckdb {duckdb_version} on {target}. " + f"This build will not load an unverified native extension into the server. Obtain the " + f"digest, confirm it independently of this download, and add it to DIGESTS in " + f"{Path(__file__).name} (`--digest` prints what the repository is currently serving)." + ) from None + + +def download(duckdb_version: str, target: str) -> bytes: + url = f"{REPOSITORY}/v{duckdb_version}/{target}/lance.duckdb_extension.gz" + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request) as response: + return gzip.decompress(response.read()) + + def check(expected: str) -> None: installed = version("duckdb") if installed != expected: @@ -47,17 +80,24 @@ def check(expected: str) -> None: def fetch(duckdb_version: str, duckdb_home: Path) -> None: - destination = duckdb_home / "extensions" / f"v{duckdb_version}" / target_platform() - destination.mkdir(parents=True, exist_ok=True) + target = target_platform() + expected = expected_digest(duckdb_version, target) - url = f"{REPOSITORY}/v{duckdb_version}/{target_platform()}/lance.duckdb_extension.gz" - request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) - with urllib.request.urlopen(request) as response: - payload = gzip.decompress(response.read()) + payload = download(duckdb_version, target) + digest = hashlib.sha256(payload).hexdigest() + if digest != expected: + # Nothing is written: the bytes are not what this image is pinned to. + raise SystemExit( + f"the lance extension served for duckdb {duckdb_version} on {target} has sha256 " + f"{digest}, but this image pins {expected}. Refusing to bake an unverified native " + f"extension." + ) + destination = duckdb_home / "extensions" / f"v{duckdb_version}" / target + destination.mkdir(parents=True, exist_ok=True) written = destination / "lance.duckdb_extension" written.write_bytes(payload) - print(f"wrote {written} ({len(payload)} bytes)") + print(f"wrote {written} ({len(payload)} bytes, sha256 {digest})") def main() -> None: @@ -65,6 +105,7 @@ def main() -> None: parser.add_argument("--duckdb-version", help="defaults to the installed duckdb") parser.add_argument("--duckdb-home", type=Path, default=Path.home() / ".duckdb") parser.add_argument("--check", action="store_true", help="assert the venv agrees with the pin") + parser.add_argument("--digest", action="store_true", help="print the served digest, for pinning") args = parser.parse_args() duckdb_version = args.duckdb_version or version("duckdb") @@ -72,6 +113,10 @@ def main() -> None: check(duckdb_version) print(f"ok: venv duckdb matches the baked lance extension ({duckdb_version})") return + if args.digest: + target = target_platform() + print(f'"{target}": "{hashlib.sha256(download(duckdb_version, target)).hexdigest()}",') + return fetch(duckdb_version, args.duckdb_home) From 59ba87dc780fff880cfb5b675608100f70cdba08 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Thu, 27 Aug 2026 14:29:56 -0700 Subject: [PATCH 07/16] fix(ocr): keep a table's caption in the table element MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses CodeRabbit on PR #249, and it is a real loss. `_captionable` matches tables as well as figures, so a caption next to a table was consumed out of the markdown stream — but the table branch emitted only `row["html"]`, which never carried it. "Table 1. Prevalence by district." disappeared from the document entirely, and a table's caption is usually the only prose saying what the table is of. The caption now folds into the markup as ``, escaped, as the first child of `` — the one position HTML allows it, so a row-window chunk can repeat it alongside the header. A table that produced no markup keeps its caption as bare text rather than dropping both. My own test is what let this through: it asserted the element *type* was `table` and never looked at the content. It now asserts the retained text, joined by cases for caption placement, escaping, and the no-markup fallback. --- .../extralit_server/contexts/ocr/elements.py | 22 ++++++++++- .../tests/unit/contexts/ocr/test_elements.py | 37 ++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/extralit-server/src/extralit_server/contexts/ocr/elements.py b/extralit-server/src/extralit_server/contexts/ocr/elements.py index ae0540951..bd1e27afc 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/elements.py +++ b/extralit-server/src/extralit_server/contexts/ocr/elements.py @@ -8,6 +8,7 @@ from __future__ import annotations +import html from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from typing import Optional @@ -88,6 +89,25 @@ def _bbox_of(row: Mapping) -> Optional[tuple[float, float, float, float]]: return tuple(float(v) for v in bbox) if bbox else None # ty: ignore[invalid-return-type] +def _table_content(markup: str, caption: str) -> str: + """Fold a table's caption into its markup, as the `" + opening = markup.find(">") + if not markup.startswith(" Optional[int]: """Index of the figure or table a caption at `index` belongs to. @@ -152,7 +172,7 @@ def elements_from_items(rows: Iterable[Mapping]) -> list[Element]: _push_heading(stack, slot, text) if label in TABLE_LABELS: - kind, content = TABLE, (row.get("html") or "") + kind, content = TABLE, _table_content(row.get("html") or "", captions.get(index, "")) elif label in PICTURE_LABELS: kind, content = FIGURE, captions.get(index, "") else: diff --git a/extralit-server/tests/unit/contexts/ocr/test_elements.py b/extralit-server/tests/unit/contexts/ocr/test_elements.py index 926d0004c..49b3c06c8 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_elements.py +++ b/extralit-server/tests/unit/contexts/ocr/test_elements.py @@ -101,7 +101,7 @@ def test_a_caption_is_absorbed_by_its_figure_rather_than_left_as_prose(self): assert [(e.type, e.content) for e in elements] == [(FIGURE, "Figure 1. A red square.")] - def test_a_caption_preceding_its_figure_is_still_absorbed(self): + def test_a_caption_preceding_its_table_is_absorbed_and_kept(self): rows = [ row(DocItemLabel.CAPTION, 0, text="Table 1. Counts."), row(DocItemLabel.TABLE, 1, html="
` element HTML has for it. + + A caption is consumed from the markdown stream by the table that owns it, so if it were not + put back here it would be dropped outright — and a table's caption is usually the only prose + saying what the table is of. + """ + if not markup: + return caption + if not caption: + return markup + escaped = f"{html.escape(caption, quote=False)}
x
"), @@ -110,6 +110,41 @@ def test_a_caption_preceding_its_figure_is_still_absorbed(self): elements = elements_from_items(rows) assert [e.type for e in elements] == [TABLE] + # Consumed from the markdown stream, so the table has to carry it or it is lost. + assert elements[0].content == ( + "
Table 1. Counts.
x
" + ) + + def test_a_table_caption_lands_where_html_allows_a_caption(self): + rows = [ + row(DocItemLabel.TABLE, 0, html="
N
"), + row(DocItemLabel.CAPTION, 1, text="Table 2. Sizes."), + ] + + content = elements_from_items(rows)[0].content + + # is only valid as the first child of . + assert content.startswith("
") + + def test_a_table_caption_is_escaped(self): + rows = [ + row(DocItemLabel.TABLE, 0, html="
Table 2. Sizes.
x
"), + row(DocItemLabel.CAPTION, 1, text="Risk & spread "), + ] + + content = elements_from_items(rows)[0].content + + assert "Risk & spread <n=42>" in content + + def test_a_caption_survives_a_table_that_produced_no_markup(self): + rows = [ + row(DocItemLabel.TABLE, 0, html=None), + row(DocItemLabel.CAPTION, 1, text="Table 3. Unparsed."), + ] + + elements = elements_from_items(rows) + + assert [(e.type, e.content) for e in elements] == [(TABLE, "Table 3. Unparsed.")] def test_a_caption_on_a_page_with_no_figure_survives_as_prose(self): rows = [ From b28265769ea8b396dee1c534f6575cadd32deccf Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Thu, 27 Aug 2026 14:43:05 -0700 Subject: [PATCH 08/16] build: advance the hf-space submodule, and keep --seed after all MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extralit-hf-space#12 merged as a05a1c0, so the pointer moves off the pin it had been stuck on for ten commits. That merge was supposed to unblock dropping `--seed` from `uv venv`, since hf-space no longer installs with pip. Measured, it does not. Removing the seed does not remove pip from the image: `pip` simply falls through to the base image's /usr/local/bin/pip, which installs into /usr/local/lib/python3.12/site-packages while `python` stays /opt/venv/bin/python. So a derived image running `pip install X` — exactly what hf-space did until yesterday — would install X somewhere the interpreter cannot see it, and fail at import rather than at install. The whole trade is 5.4 MB of 1847. Not worth handing that to the next person who extends this image, so `--seed` stays and the comment now records the real reason rather than a dependency that no longer exists. --- extralit-hf-space | 2 +- extralit-server/docker/server/Dockerfile | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/extralit-hf-space b/extralit-hf-space index aa6077d8b..a05a1c01b 160000 --- a/extralit-hf-space +++ b/extralit-hf-space @@ -1 +1 @@ -Subproject commit aa6077d8b300cbeaca99482a67c01e4af9770b8e +Subproject commit a05a1c01b77303a69a9ccc34ef4665255af460b5 diff --git a/extralit-server/docker/server/Dockerfile b/extralit-server/docker/server/Dockerfile index 3cd2a96b5..2f640904d 100644 --- a/extralit-server/docker/server/Dockerfile +++ b/extralit-server/docker/server/Dockerfile @@ -30,7 +30,9 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# --seed: the HF-Space image derives from this one and installs into this venv with `pip`. +# --seed, though nothing here needs pip: without it `pip` falls through to the base image's +# /usr/local/bin/pip, which installs where /opt/venv/bin/python cannot see it. Dropping it saves +# 5 MB of 1847 and hands any derived image a silent wrong-interpreter install. RUN uv venv --seed "$VIRTUAL_ENV" # The wheel is the only input that differs between two builds of the same tree, so it enters From c6568d327c7fc565a7710db6047bd1a35155a1d6 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Mon, 31 Aug 2026 00:22:21 -0700 Subject: [PATCH 09/16] refactor(ocr): project elements columnar, as one DuckDB statement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `elements_from_items` walked the rows in Python to build breadcrumbs, bind captions and render markdown — pulling every document's text out of Arrow to make strings that went straight back into it. The projection is now `elements_sql`, one statement over the `items` columns, and `elements_table` returns Arrow conforming to ELEMENT_SCHEMA. The stack-based breadcrumb becomes three window functions per heading slot: a slot is in scope only while the newest heading at or above it is still its own. Caption binding becomes lag/lead over the document window, and the markdown/table/figure split a CASE. Everything partitions by document_id, so a whole workspace projects in the pass that used to do one document — 80k rows go 1.58s -> 0.32s, and the same query can run against a Lance dataset with the scan filter pushed down. Verified against the previous implementation on 400 randomised documents (captions, page-spanning provenance, blank text, every label): identical output. One deliberate divergence — heading levels past 6 now share the deepest slot instead of nesting, since both render as `######` anyway. The `Element` dataclass is gone; callers read columns. Nothing consumed it yet. --- .../extralit_server/contexts/ocr/elements.py | 384 ++++++++++-------- .../tests/unit/contexts/ocr/test_elements.py | 206 +++++++--- 2 files changed, 372 insertions(+), 218 deletions(-) diff --git a/extralit-server/src/extralit_server/contexts/ocr/elements.py b/extralit-server/src/extralit_server/contexts/ocr/elements.py index bd1e27afc..e3dff4117 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/elements.py +++ b/extralit-server/src/extralit_server/contexts/ocr/elements.py @@ -2,21 +2,28 @@ `arrow.item_rows` flattens a `DoclingDocument` into columns; this reads those columns back as the three units a chunker dispatches on — a markdown run, a table, or a figure with its caption -— so `contexts/retrieval` never has to know a docling label. Rows are the only input, which is -what lets chunking re-run from the Lance dataset without re-parsing the PDF. +— so `contexts/retrieval` never has to know a docling label. Rows in, rows out: chunking re-runs +from the Lance dataset without re-parsing the PDF. + +The whole projection is one DuckDB statement over the `items` columns. Reading breadcrumbs and +captions out row by row would mean pulling every document's text into Python to build strings +that go straight back into Arrow; as SQL it stays columnar, pushes the projection down to Lance, +and does a whole workspace in the same pass as a single document. """ from __future__ import annotations -import html -from collections.abc import Iterable, Mapping, Sequence -from dataclasses import dataclass -from typing import Optional +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, Optional +import pyarrow as pa from docling_core.types.doc import DocItemLabel from extralit_server.contexts.ocr.docling_builder import PICTURE_LABELS, TABLE_LABELS +if TYPE_CHECKING: + import duckdb + MARKDOWN = "markdown" TABLE = "table" FIGURE = "figure" @@ -27,172 +34,207 @@ #: Headings open a breadcrumb slot. A title sits above every section header, whatever its level. TITLE_SLOT = 0 -_MAX_HEADING_LEVEL = 6 - - -@dataclass(frozen=True) -class Element: - """One retrievable unit, carrying the provenance a hit needs to point back at the page.""" - - type: str - content: str - page_no: Optional[int] - bbox: Optional[tuple[float, float, float, float]] - label: str - level: Optional[int] - item_ref: str - reading_order: int - headings: tuple[str, ...] = () - - -def _text_of(row: Mapping, siblings: int) -> str: - """A row's own text, sliced to its provenance when one item spans several of them.""" - text = row.get("text") or "" - if siblings < 2: - return text - start, end = row.get("charspan_start"), row.get("charspan_end") - if start is None or end is None or end <= start: - return text - return text[start:end] - - -def _markdown(label: str, text: str, level: Optional[int]) -> str: - """Render a row as the markdown the recursive chunker's rules are written against.""" - if label == DocItemLabel.TITLE: - return f"# {text}" - if label == DocItemLabel.SECTION_HEADER: - return "#" * min(max(level or 1, 1), _MAX_HEADING_LEVEL) + f" {text}" - if label == DocItemLabel.LIST_ITEM: - return f"- {text}" - if label == DocItemLabel.CODE: - return f"```\n{text}\n```" - return text - - -def _heading_slot(label: str, level: Optional[int]) -> Optional[int]: - if label == DocItemLabel.TITLE: - return TITLE_SLOT - if label == DocItemLabel.SECTION_HEADER: - return max(level or 1, 1) - return None - - -def _push_heading(stack: list[tuple[int, str]], slot: int, text: str) -> None: - """Open a breadcrumb slot, closing every slot at or below it.""" - while stack and stack[-1][0] >= slot: - stack.pop() - stack.append((slot, text)) - - -def _bbox_of(row: Mapping) -> Optional[tuple[float, float, float, float]]: - bbox = row.get("bbox") - return tuple(float(v) for v in bbox) if bbox else None # ty: ignore[invalid-return-type] +#: Both the deepest renderable heading and the deepest breadcrumb slot; `######` has no successor. +MAX_HEADING_LEVEL = 6 + +#: How far either side of a caption its figure or table may sit, nearest first, before after. +CAPTION_REACH = (-1, 1, -2, 2) + +ELEMENT_SCHEMA = pa.schema( + [ + ("document_id", pa.string()), + ("type", pa.string()), + ("content", pa.string()), + ("page_no", pa.int32()), + ("bbox", pa.list_(pa.float32(), 4)), + ("label", pa.string()), + ("level", pa.int8()), + ("item_ref", pa.string()), + ("reading_order", pa.int32()), + ("headings", pa.list_(pa.string())), + ("heading_level", pa.int8()), + ] +) + + +def _labels(labels: Iterable[DocItemLabel]) -> str: + """Render a label set as a SQL `IN` list. Enum values only — nothing here is caller input.""" + return "(" + ", ".join(f"'{label.value}'" for label in sorted(labels)) + ")" + + +#: Python's `str.strip`; DuckDB's bare `trim` takes spaces off and leaves newlines behind. +_STRIP = r"regexp_replace({0}, '^\s+|\s+$', '', 'g')" + +_RUNNING = "PARTITION BY document_id ORDER BY ord ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW" + +_HEADING_LEVEL = f"least(greatest(coalesce(level, 1), 1), {MAX_HEADING_LEVEL})" + + +def _breadcrumb_columns() -> str: + """One text and two positions per slot: a slot is in scope only if nothing has closed it.""" + parts = [] + for slot in range(TITLE_SLOT, MAX_HEADING_LEVEL + 1): + parts += [ + f"last_value(CASE WHEN slot = {slot} THEN own_text END IGNORE NULLS) OVER running AS heading_{slot}", + f"max(CASE WHEN slot = {slot} THEN ord END) OVER running AS opened_{slot}", + f"max(CASE WHEN slot <= {slot} THEN ord END) OVER running AS covered_{slot}", + ] + return ",\n ".join(parts) + + +def _breadcrumb_list() -> str: + """A slot survives only while the newest heading at or above it is still its own.""" + slots = ", ".join( + f"CASE WHEN opened_{slot} = covered_{slot} THEN heading_{slot} END" + for slot in range(TITLE_SLOT, MAX_HEADING_LEVEL + 1) + ) + return f"list_filter(list_value({slots}), heading -> heading IS NOT NULL)" + + +def _caption_owner() -> str: + """The figure or table a caption binds to: nearest on the same page, looking back first.""" + branches = [] + for offset in CAPTION_REACH: + window, distance = ("lag" if offset < 0 else "lead"), abs(offset) + neighbour = f"{window}({{0}}, {distance}) OVER document" + branches.append( + f"WHEN {neighbour.format('label')} IN {_labels(TABLE_LABELS | PICTURE_LABELS)}" + f" AND {neighbour.format('page_no')} IS NOT DISTINCT FROM page_no" + f" THEN {neighbour.format('ord')}" + ) + return ( + "CASE WHEN label <> 'caption' OR own_text = '' THEN NULL\n " + + "\n ".join(branches) + + "\n END" + ) + + +#: A caption is consumed by its owner, so a table that drops it drops the only prose naming it. +_CAPTION_TAG = ( + "'' || replace(replace(replace(caption, '&', '&'), '<', '<'), '>', '>') || ''" +) + +_TABLE_CONTENT = f"""CASE + WHEN coalesce(html, '') = '' THEN coalesce(caption, '') + WHEN coalesce(caption, '') = '' THEN html + WHEN starts_with(html, '' IN html) > 0 + THEN substr(html, 1, position('>' IN html)) || {_CAPTION_TAG} + || substr(html, position('>' IN html) + 1) + ELSE {_CAPTION_TAG} || html + END""" + +_CONTENT = f"""CASE + WHEN label IN {_labels(TABLE_LABELS)} THEN {_TABLE_CONTENT} + WHEN label IN {_labels(PICTURE_LABELS)} THEN coalesce(caption, '') + WHEN own_text = '' THEN '' + WHEN label = '{DocItemLabel.TITLE.value}' THEN '# ' || own_text + WHEN label = '{DocItemLabel.SECTION_HEADER.value}' + THEN repeat('#', {_HEADING_LEVEL}) || ' ' || own_text + WHEN label = '{DocItemLabel.LIST_ITEM.value}' THEN '- ' || own_text + WHEN label = '{DocItemLabel.CODE.value}' THEN '```' || chr(10) || own_text || chr(10) || '```' + ELSE own_text + END""" + + +def elements_sql(source: str = "items") -> str: + """The projection, as one statement over anything DuckDB can scan with the `items` columns.""" + return f""" +WITH ordered AS ( + SELECT + document_id, self_ref, level, reading_order, page_no, bbox, html, text, + charspan_start, charspan_end, + coalesce(label, '{DocItemLabel.TEXT.value}') AS label, + row_number() OVER document AS ord, + count(*) OVER (PARTITION BY document_id, self_ref) AS provs + FROM {source} + WINDOW document AS (PARTITION BY document_id ORDER BY reading_order, prov_index) +), +sliced AS ( + SELECT * EXCLUDE (text, charspan_start, charspan_end, provs), + { + _STRIP.format( + '''CASE + WHEN provs >= 2 AND charspan_start IS NOT NULL AND charspan_end IS NOT NULL + AND charspan_end > charspan_start + THEN substr(coalesce(text, ''), charspan_start + 1, charspan_end - charspan_start) + ELSE coalesce(text, '') + END''' + ) + } AS own_text + FROM ordered +), +slotted AS ( + SELECT *, + CASE + WHEN own_text = '' THEN NULL + WHEN label = '{DocItemLabel.TITLE.value}' THEN {TITLE_SLOT} + WHEN label = '{DocItemLabel.SECTION_HEADER.value}' THEN {_HEADING_LEVEL} + END AS slot + FROM sliced +), +breadcrumbs AS ( + SELECT *, + {_breadcrumb_columns()}, + last_value(slot IGNORE NULLS) OVER running AS heading_level + FROM slotted + WINDOW running AS ({_RUNNING}) +), +neighbours AS ( + SELECT * EXCLUDE (slot), + {_breadcrumb_list()} AS headings, + {_caption_owner()} AS owner_ord + FROM breadcrumbs + WINDOW document AS (PARTITION BY document_id ORDER BY ord) +), +captions AS ( + SELECT document_id, owner_ord, string_agg(own_text, ' ' ORDER BY ord) AS caption + FROM neighbours + WHERE owner_ord IS NOT NULL + GROUP BY document_id, owner_ord +), +projected AS ( + SELECT + element.document_id, + element.ord, + CASE + WHEN label IN {_labels(TABLE_LABELS)} THEN '{TABLE}' + WHEN label IN {_labels(PICTURE_LABELS)} THEN '{FIGURE}' + ELSE '{MARKDOWN}' + END AS type, + {_CONTENT} AS content, + element.page_no, element.bbox, element.label, element.level, + element.self_ref AS item_ref, element.reading_order, + element.headings, element.heading_level + FROM neighbours AS element + LEFT JOIN captions + ON captions.document_id = element.document_id AND captions.owner_ord = element.ord + WHERE label NOT IN {_labels(SKIPPED_LABELS)} + AND element.owner_ord IS NULL +) +SELECT document_id, type, content, page_no, bbox, label, level, item_ref, reading_order, + headings, heading_level +FROM projected +WHERE content <> '' +ORDER BY document_id, ord +""" -def _table_content(markup: str, caption: str) -> str: - """Fold a table's caption into its markup, as the `` element HTML has for it. +def elements_table(items: Any, *, connection: Optional[duckdb.DuckDBPyConnection] = None) -> pa.Table: + """Project `items` rows into elements, in reading order, one per provenance row. - A caption is consumed from the markdown stream by the table that owns it, so if it were not - put back here it would be dropped outright — and a table's caption is usually the only prose - saying what the table is of. - """ - if not markup: - return caption - if not caption: - return markup - escaped = f"{html.escape(caption, quote=False)}" - opening = markup.find(">") - if not markup.startswith(" Optional[int]: - """Index of the figure or table a caption at `index` belongs to. - - docling puts a caption after its figure, but parsers that sort geometrically can put it - either side, so the nearer neighbour on the same page wins. - """ - page = rows[index].get("page_no") - best: Optional[tuple[int, int]] = None - for offset in (-1, 1, -2, 2): - neighbour = index + offset - if not 0 <= neighbour < len(rows): - continue - row = rows[neighbour] - if row.get("page_no") != page: - continue - if row.get("label") in TABLE_LABELS or row.get("label") in PICTURE_LABELS: - distance = abs(offset) - if best is None or distance < best[0]: - best = (distance, neighbour) - return best[1] if best else None - - -def elements_from_items(rows: Iterable[Mapping]) -> list[Element]: - """Read `items` rows back into elements, in reading order. - - One element per row, so an item spanning a page break stays two elements with two bboxes - rather than one element that claims to be in two places. + An item spanning a page break stays two elements with two bboxes rather than one element + claiming to be in two places. `items` is anything DuckDB registers — an Arrow table, a Lance + dataset — and every document in it is projected in the same pass. """ - ordered = sorted(rows, key=lambda r: (r.get("reading_order") or 0, r.get("prov_index") or 0)) - provs: dict[str, int] = {} - for row in ordered: - provs[row.get("self_ref")] = provs.get(row.get("self_ref"), 0) + 1 - - # Captions are consumed by the figure or table they describe, so resolve them first. - captions: dict[int, str] = {} - consumed: set[int] = set() - for index, row in enumerate(ordered): - if row.get("label") != DocItemLabel.CAPTION: - continue - owner = _captionable(ordered, index) - if owner is None: - continue - text = _text_of(row, provs.get(row.get("self_ref"), 1)).strip() - if not text: - continue - captions[owner] = f"{captions[owner]} {text}" if owner in captions else text - consumed.add(index) - - elements: list[Element] = [] - stack: list[tuple[int, str]] = [] - - for index, row in enumerate(ordered): - label = row.get("label") or DocItemLabel.TEXT - if label in SKIPPED_LABELS or index in consumed: - continue - - level = row.get("level") - text = _text_of(row, provs.get(row.get("self_ref"), 1)).strip() - - slot = _heading_slot(label, level) - if slot is not None and text: - _push_heading(stack, slot, text) - - if label in TABLE_LABELS: - kind, content = TABLE, _table_content(row.get("html") or "", captions.get(index, "")) - elif label in PICTURE_LABELS: - kind, content = FIGURE, captions.get(index, "") - else: - kind, content = MARKDOWN, _markdown(label, text, level) if text else "" - - if not content: - continue - - elements.append( - Element( - type=kind, - content=content, - page_no=row.get("page_no"), - bbox=_bbox_of(row), - label=str(label), - level=level, - item_ref=row.get("self_ref"), - reading_order=row.get("reading_order") or 0, - headings=tuple(heading for _, heading in stack), - ) - ) - - return elements + import duckdb + + own = connection is None + connection = connection or duckdb.connect() + try: + connection.register("_items", items) + table = connection.execute(elements_sql("_items")).to_arrow_table() + finally: + connection.unregister("_items") + if own: + connection.close() + return table.cast(ELEMENT_SCHEMA) diff --git a/extralit-server/tests/unit/contexts/ocr/test_elements.py b/extralit-server/tests/unit/contexts/ocr/test_elements.py index 49b3c06c8..c7258396a 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_elements.py +++ b/extralit-server/tests/unit/contexts/ocr/test_elements.py @@ -1,16 +1,23 @@ -"""Tests for the typed element view of `items` rows.""" +"""Tests for the columnar element view of `items` rows.""" +import pyarrow as pa import pytest from docling_core.types.doc import BoundingBox, CoordOrigin, DocItemLabel, Size -from extralit_server.contexts.ocr.arrow import item_rows, table_html +from extralit_server.contexts.ocr.arrow import ITEM_SCHEMA, item_rows, table_html from extralit_server.contexts.ocr.docling_builder import ( LayoutBlock, PageContext, append_blocks, new_document, ) -from extralit_server.contexts.ocr.elements import FIGURE, MARKDOWN, TABLE, elements_from_items +from extralit_server.contexts.ocr.elements import ( + ELEMENT_SCHEMA, + FIGURE, + MARKDOWN, + TABLE, + elements_table, +) from extralit_server.contexts.ocr.tables import make_cell DOCUMENT_ID = "11111111-2222-3333-4444-555555555555" @@ -22,13 +29,17 @@ def bbox(t: float, b: float, left: float = 10.0, right: float = 100.0) -> Boundi def row(label, reading_order, **overrides): base = { + "document_id": DOCUMENT_ID, "self_ref": f"#/texts/{reading_order}", + "parent_ref": None, "label": label, + "content_layer": "body", "level": None, "reading_order": reading_order, "prov_index": 0, "page_no": 1, "bbox": [0.0, 0.0, 10.0, 10.0], + "coord_origin": "TOPLEFT", "text": None, "html": None, "charspan_start": None, @@ -38,6 +49,24 @@ def row(label, reading_order, **overrides): return base +def elements(rows) -> list[dict]: + """The projection as dicts — assertions read better than Arrow column slices.""" + return elements_table(pa.Table.from_pylist(list(rows), schema=ITEM_SCHEMA)).to_pylist() + + +class TestSchema: + def test_the_projection_conforms_to_the_declared_schema(self): + table = elements_table(pa.Table.from_pylist([row(DocItemLabel.TEXT, 0, text="x")], schema=ITEM_SCHEMA)) + + assert table.schema == ELEMENT_SCHEMA + + def test_an_empty_input_yields_an_empty_table_not_an_error(self): + table = elements_table(pa.Table.from_pylist([], schema=ITEM_SCHEMA)) + + assert table.num_rows == 0 + assert table.schema == ELEMENT_SCHEMA + + class TestHeadingBreadcrumb: def test_breadcrumb_deepens_and_unwinds_with_heading_level(self): rows = [ @@ -49,22 +78,52 @@ def test_breadcrumb_deepens_and_unwinds_with_heading_level(self): row(DocItemLabel.TEXT, 5, text="Shallow body."), ] - elements = elements_from_items(rows) - - assert [e.headings for e in elements] == [ - ("A Paper",), - ("A Paper", "Methods"), - ("A Paper", "Methods", "Sampling"), - ("A Paper", "Methods", "Sampling"), + assert [e["headings"] for e in elements(rows)] == [ + ["A Paper"], + ["A Paper", "Methods"], + ["A Paper", "Methods", "Sampling"], + ["A Paper", "Methods", "Sampling"], # Results closes Methods and Sampling but never the title. - ("A Paper", "Results"), - ("A Paper", "Results"), + ["A Paper", "Results"], + ["A Paper", "Results"], ] def test_a_heading_carries_itself_so_a_chunk_knows_its_own_path(self): rows = [row(DocItemLabel.SECTION_HEADER, 0, level=1, text="Methods")] - assert elements_from_items(rows)[0].headings == ("Methods",) + assert elements(rows)[0]["headings"] == ["Methods"] + + def test_heading_level_tracks_the_deepest_slot_in_scope(self): + rows = [ + row(DocItemLabel.TITLE, 0, text="A Paper"), + row(DocItemLabel.SECTION_HEADER, 1, level=2, text="Sampling"), + row(DocItemLabel.TEXT, 2, text="Body."), + ] + + assert [e["heading_level"] for e in elements(rows)] == [0, 2, 2] + + def test_prose_before_any_heading_has_no_breadcrumb(self): + element = elements([row(DocItemLabel.TEXT, 0, text="Orphan.")])[0] + assert element["headings"] == [] + assert element["heading_level"] is None + + def test_a_blank_heading_opens_no_slot(self): + rows = [ + row(DocItemLabel.SECTION_HEADER, 0, level=1, text=" "), + row(DocItemLabel.TEXT, 1, text="Body."), + ] + + assert [e["headings"] for e in elements(rows)] == [[]] + + def test_levels_past_h6_share_the_deepest_slot(self): + # Both render as `######`, so nesting one under the other would be a distinction + # the markdown cannot carry. + rows = [ + row(DocItemLabel.SECTION_HEADER, 0, level=7, text="Seven"), + row(DocItemLabel.SECTION_HEADER, 1, level=99, text="Ninety-nine"), + ] + + assert [e["headings"] for e in elements(rows)] == [["Seven"], ["Ninety-nine"]] class TestMarkdownRendering: @@ -76,18 +135,20 @@ class TestMarkdownRendering: (DocItemLabel.SECTION_HEADER, None, "Unlevelled", "# Unlevelled"), (DocItemLabel.LIST_ITEM, None, "first", "- first"), (DocItemLabel.TEXT, None, "Prose.", "Prose."), + (DocItemLabel.CODE, None, "x = 1", "```\nx = 1\n```"), ], ) def test_rows_render_as_the_markdown_the_chunker_rules_expect(self, label, level, text, expected): - elements = elements_from_items([row(label, 0, level=level, text=text)]) + element = elements([row(label, 0, level=level, text=text)])[0] - assert elements[0].type == MARKDOWN - assert elements[0].content == expected + assert element["type"] == MARKDOWN + assert element["content"] == expected def test_heading_level_is_clamped_to_six(self): - elements = elements_from_items([row(DocItemLabel.SECTION_HEADER, 0, level=99, text="Deep")]) + assert elements([row(DocItemLabel.SECTION_HEADER, 0, level=99, text="Deep")])[0]["content"] == "###### Deep" - assert elements[0].content == "###### Deep" + def test_surrounding_whitespace_is_stripped_including_newlines(self): + assert elements([row(DocItemLabel.TEXT, 0, text=" \n\tProse.\n ")])[0]["content"] == "Prose." class TestCaptions: @@ -97,9 +158,7 @@ def test_a_caption_is_absorbed_by_its_figure_rather_than_left_as_prose(self): row(DocItemLabel.CAPTION, 1, text="Figure 1. A red square."), ] - elements = elements_from_items(rows) - - assert [(e.type, e.content) for e in elements] == [(FIGURE, "Figure 1. A red square.")] + assert [(e["type"], e["content"]) for e in elements(rows)] == [(FIGURE, "Figure 1. A red square.")] def test_a_caption_preceding_its_table_is_absorbed_and_kept(self): rows = [ @@ -107,11 +166,11 @@ def test_a_caption_preceding_its_table_is_absorbed_and_kept(self): row(DocItemLabel.TABLE, 1, html="
x
"), ] - elements = elements_from_items(rows) + found = elements(rows) - assert [e.type for e in elements] == [TABLE] + assert [e["type"] for e in found] == [TABLE] # Consumed from the markdown stream, so the table has to carry it or it is lost. - assert elements[0].content == ( + assert found[0]["content"] == ( "
Table 1. Counts.
x
" ) @@ -121,10 +180,8 @@ def test_a_table_caption_lands_where_html_allows_a_caption(self): row(DocItemLabel.CAPTION, 1, text="Table 2. Sizes."), ] - content = elements_from_items(rows)[0].content - # is only valid as the first child of . - assert content.startswith("
") + assert elements(rows)[0]["content"].startswith("
Table 2. Sizes.
") def test_a_table_caption_is_escaped(self): rows = [ @@ -132,9 +189,7 @@ def test_a_table_caption_is_escaped(self): row(DocItemLabel.CAPTION, 1, text="Risk & spread "), ] - content = elements_from_items(rows)[0].content - - assert "" in content + assert "" in elements(rows)[0]["content"] def test_a_caption_survives_a_table_that_produced_no_markup(self): rows = [ @@ -142,9 +197,15 @@ def test_a_caption_survives_a_table_that_produced_no_markup(self): row(DocItemLabel.CAPTION, 1, text="Table 3. Unparsed."), ] - elements = elements_from_items(rows) + assert [(e["type"], e["content"]) for e in elements(rows)] == [(TABLE, "Table 3. Unparsed.")] + + def test_a_caption_keeps_markup_that_is_not_a_table_element(self): + rows = [ + row(DocItemLabel.TABLE, 0, html="not-a-table"), + row(DocItemLabel.CAPTION, 1, text="Table 4. Odd."), + ] - assert [(e.type, e.content) for e in elements] == [(TABLE, "Table 3. Unparsed.")] + assert elements(rows)[0]["content"] == "not-a-table" def test_a_caption_on_a_page_with_no_figure_survives_as_prose(self): rows = [ @@ -152,12 +213,34 @@ def test_a_caption_on_a_page_with_no_figure_survives_as_prose(self): row(DocItemLabel.CAPTION, 1, page_no=1, text="Orphaned."), ] - elements = elements_from_items(rows) + assert [(e["type"], e["content"]) for e in elements(rows)] == [(MARKDOWN, "Orphaned.")] - assert [(e.type, e.content) for e in elements] == [(MARKDOWN, "Orphaned.")] + def test_the_nearer_neighbour_wins_when_a_caption_sits_between_two_figures(self): + rows = [ + row(DocItemLabel.PICTURE, 0), + row(DocItemLabel.CAPTION, 1, text="Belongs to the first."), + row(DocItemLabel.TEXT, 2, text="Prose."), + row(DocItemLabel.PICTURE, 3), + ] + + found = elements(rows) + + assert [(e["type"], e["content"]) for e in found] == [ + (FIGURE, "Belongs to the first."), + (MARKDOWN, "Prose."), + ] + + def test_two_captions_on_one_figure_are_joined(self): + rows = [ + row(DocItemLabel.CAPTION, 0, text="Figure 1."), + row(DocItemLabel.PICTURE, 1), + row(DocItemLabel.CAPTION, 2, text="A red square."), + ] + + assert [(e["type"], e["content"]) for e in elements(rows)] == [(FIGURE, "Figure 1. A red square.")] def test_an_uncaptioned_figure_yields_nothing_retrievable(self): - assert elements_from_items([row(DocItemLabel.PICTURE, 0)]) == [] + assert elements([row(DocItemLabel.PICTURE, 0)]) == [] class TestProvenance: @@ -168,7 +251,7 @@ def test_running_headers_and_footers_are_dropped(self): row(DocItemLabel.PAGE_FOOTER, 2, text="7"), ] - assert [e.content for e in elements_from_items(rows)] == ["Real body."] + assert [e["content"] for e in elements(rows)] == ["Real body."] def test_one_element_per_provenance_row_when_an_item_spans_a_page_break(self): spanning = [ @@ -184,20 +267,49 @@ def test_one_element_per_provenance_row_when_an_item_spans_a_page_break(self): ), ] - elements = elements_from_items(spanning) - - assert [(e.page_no, e.content) for e in elements] == [(1, "first half"), (2, "second half")] + assert [(e["page_no"], e["content"]) for e in elements(spanning)] == [(1, "first half"), (2, "second half")] def test_elements_come_back_in_reading_order_whatever_the_row_order(self): rows = [row(DocItemLabel.TEXT, 2, text="third"), row(DocItemLabel.TEXT, 0, text="first")] - assert [e.content for e in elements_from_items(rows)] == ["first", "third"] + assert [e["content"] for e in elements(rows)] == ["first", "third"] def test_bbox_and_item_ref_survive_the_round_trip(self): - elements = elements_from_items([row(DocItemLabel.TEXT, 0, text="x", bbox=[1.0, 2.0, 3.0, 4.0])]) + element = elements([row(DocItemLabel.TEXT, 0, text="x", bbox=[1.0, 2.0, 3.0, 4.0])])[0] + + assert element["bbox"] == [1.0, 2.0, 3.0, 4.0] + assert element["item_ref"] == "#/texts/0" + + +class TestManyDocuments: + def test_documents_are_projected_in_one_pass_without_bleeding_into_each_other(self): + rows = [] + for document_id in ("aaaa", "bbbb"): + for base in ( + row(DocItemLabel.SECTION_HEADER, 0, level=1, text=f"{document_id} heading"), + row(DocItemLabel.TEXT, 1, text=f"{document_id} body"), + ): + rows.append({**base, "document_id": document_id}) + + found = elements(reversed(rows)) - assert elements[0].bbox == (1.0, 2.0, 3.0, 4.0) - assert elements[0].item_ref == "#/texts/0" + assert [(e["document_id"], e["headings"]) for e in found] == [ + ("aaaa", ["aaaa heading"]), + ("aaaa", ["aaaa heading"]), + ("bbbb", ["bbbb heading"]), + ("bbbb", ["bbbb heading"]), + ] + + def test_a_caption_never_binds_across_a_document_boundary(self): + rows = [ + {**row(DocItemLabel.PICTURE, 0), "document_id": "aaaa"}, + {**row(DocItemLabel.CAPTION, 0, text="Belongs to bbbb."), "document_id": "bbbb"}, + ] + + # The figure stays uncaptioned and drops out; the caption stays prose in its own document. + assert [(e["document_id"], e["type"], e["content"]) for e in elements(rows)] == [ + ("bbbb", MARKDOWN, "Belongs to bbbb.") + ] class TestTableHtml: @@ -286,8 +398,8 @@ def test_elements_line_up_with_a_projected_document(self): ], ) - elements = elements_from_items(item_rows(document, DOCUMENT_ID)) + found = elements(item_rows(document, DOCUMENT_ID)) - assert [e.type for e in elements] == [MARKDOWN, MARKDOWN, TABLE] - assert all(e.headings == ("Methods",) for e in elements) - assert "" in elements[-1].content + assert [e["type"] for e in found] == [MARKDOWN, MARKDOWN, TABLE] + assert all(e["headings"] == ["Methods"] for e in found) + assert "" in found[-1]["content"] From 5af997ba40559639be1530b41ffeb07f39761e4c Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Thu, 3 Sep 2026 00:20:02 -0700 Subject: [PATCH 10/16] refactor(ocr): delete the docling reimplementations, render elements with docling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three things on this branch duplicated docling-core: an own ``/`` table serializer in arrow.py, a caption-to-figure geometry join and a label→markdown CASE in elements.py. All deleted. What replaces them, and what was checked (docling-core 2.91, chonkie 1.7): - `append_blocks` now links each caption to the nearest table/picture on its page (`FloatingItem.captions`, back first: -1, +1, -2, +2). The parsers never populated `.captions`, which is the only reason the association was rebuilt downstream by geometry. - `items` gains a `markdown` column: `MarkdownDocSerializer.serialize(item=…)` per item, with `escape_html=False`, `escape_underscores=False`, `image_placeholder=""` so a search index sees raw characters. A linked caption renders as `''` on its own and as a prefix of its owner, so it appears exactly once and the existing `content <> ''` filter drops it. - `elements_sql` keeps only the breadcrumb windows, which docling gates behind the `chunking` extra (`transformers`). Content is `markdown`, except a page-spanning text item, which falls back to its charspan slice of `text`; NULL `markdown` (rows written before the column existed) falls back to `text` too. - `html` stays and reverts to `item.export_to_html(doc=doc)`: it feeds the layout API (`LayoutItemOut.html`, OpenAPI contract) and keeps spans for the viewer; nothing chunks it. - `LayoutStore._replace_one` widens a dataset written under an older schema with NULL columns via `add_columns`, so existing workspace datasets accept the appended rows. Intentional behaviour changes: section headers render one level deeper (`## Methods` for level 1, docling reserving `#` for the title); table content is caption + pipe markdown, not `
Table 2. Sizes.Risk & spread <n=42>Risk & spread <n=42>Table 4. Odd.
` HTML; heading depth is no longer clamped at six in the content (the breadcrumb slot still is). `TableChunker(chunk_size=2)` repeating caption + header on every chunk is now pinned by a test, since we rely on chonkie for it. --- .../src/extralit_server/contexts/ocr/arrow.py | 67 ++-- .../contexts/ocr/docling_builder.py | 29 +- .../extralit_server/contexts/ocr/elements.py | 144 +++------ .../contexts/ocr/layout_store.py | 21 ++ .../tests/unit/contexts/ocr/test_arrow.py | 29 ++ .../unit/contexts/ocr/test_docling_builder.py | 74 +++++ .../tests/unit/contexts/ocr/test_elements.py | 295 +++++++----------- .../unit/contexts/ocr/test_layout_store.py | 13 + 8 files changed, 337 insertions(+), 335 deletions(-) diff --git a/extralit-server/src/extralit_server/contexts/ocr/arrow.py b/extralit-server/src/extralit_server/contexts/ocr/arrow.py index 501f081eb..d8412748e 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/arrow.py +++ b/extralit-server/src/extralit_server/contexts/ocr/arrow.py @@ -7,12 +7,11 @@ from __future__ import annotations -import html -from collections import defaultdict from typing import Any, Optional import pyarrow as pa -from docling_core.types.doc import DoclingDocument, TableCell +from docling_core.transforms.serializer.markdown import MarkdownDocSerializer, MarkdownParams +from docling_core.types.doc import DoclingDocument from docling_core.types.doc.document import DocItem, TableItem ITEM_SCHEMA = pa.schema( @@ -31,6 +30,7 @@ ("charspan_start", pa.int32()), ("charspan_end", pa.int32()), ("text", pa.string()), + ("markdown", pa.string()), ("html", pa.string()), ] ) @@ -51,62 +51,34 @@ def _enum_value(value: Any) -> Optional[str]: return getattr(value, "value", value) -def _cell_html(cell: TableCell, in_header: bool) -> str: - tag = "th" if in_header or cell.column_header or cell.row_header else "td" - spans = "" - if cell.col_span > 1: - spans += f' colspan="{cell.col_span}"' - if cell.row_span > 1: - spans += f' rowspan="{cell.row_span}"' - return f"<{tag}{spans}>{html.escape(cell.text or '', quote=False)}" - - -def table_html(item: TableItem) -> Optional[str]: - """Serialize a table with a real ``. - - docling's own exporter puts the header's ``, which leaves the - header unliftable — and a table split into row windows has to repeat it on every chunk. - """ - cells = list(getattr(item.data, "table_cells", None) or []) - if not cells: +def _table_html(doc: DoclingDocument, item: DocItem) -> Optional[str]: + if not isinstance(item, TableItem): + return None + try: + return item.export_to_html(doc=doc) or None + except Exception: # a malformed table must not sink the whole projection return None - by_row: dict[int, list[TableCell]] = defaultdict(list) - for cell in cells: - by_row[cell.start_row_offset_idx].append(cell) - rows = [sorted(by_row[index], key=lambda c: c.start_col_offset_idx) for index in sorted(by_row)] - - # Only a leading run of all-header rows is a header; a stray `column_header` further down - # is a mislabelled body cell, not the start of a second one. - header_rows = 0 - for row in rows: - if not all(cell.column_header for cell in row): - break - header_rows += 1 - - def section(tag: str, group: list[list[TableCell]], in_header: bool) -> str: - if not group: - return "" - body = "".join("" + "".join(_cell_html(c, in_header) for c in row) + "" for row in group) - return f"<{tag}>{body}" - head = section("thead", rows[:header_rows], True) - body = section("tbody", rows[header_rows:], False) - return f"
` cells inside `
{head}{body}
" +def markdown_serializer(doc: DoclingDocument) -> MarkdownDocSerializer: + """docling's renderer, tuned for retrieval: raw characters, no image placeholder.""" + return MarkdownDocSerializer( + doc=doc, + params=MarkdownParams(escape_html=False, escape_underscores=False, image_placeholder=""), + ) -def _table_html(doc: DoclingDocument, item: DocItem) -> Optional[str]: - if not isinstance(item, TableItem): - return None +def _item_markdown(serializer: MarkdownDocSerializer, item: DocItem) -> Optional[str]: try: - return table_html(item) - except Exception: # a malformed table must not sink the whole projection + return serializer.serialize(item=item).text + except Exception: # NULL, not '': a row the renderer could not handle is not a blank one return None def item_rows(doc: DoclingDocument, document_id: str) -> list[dict[str, Any]]: """Flatten every item's provenance into rows matching `ITEM_SCHEMA`.""" rows: list[dict[str, Any]] = [] + serializer = markdown_serializer(doc) for reading_order, (item, _level) in enumerate(doc.iterate_items(with_groups=False)): base = { @@ -118,6 +90,7 @@ def item_rows(doc: DoclingDocument, document_id: str) -> list[dict[str, Any]]: "level": getattr(item, "level", None), "reading_order": reading_order, "text": getattr(item, "text", None) or None, + "markdown": _item_markdown(serializer, item), "html": _table_html(doc, item), } diff --git a/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py b/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py index cf2bc03b0..451d3ac49 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py +++ b/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py @@ -20,7 +20,7 @@ Size, TableCell, ) -from docling_core.types.doc.document import DocumentOrigin, NodeItem +from docling_core.types.doc.document import DocumentOrigin, FloatingItem, NodeItem, PictureItem, TableItem PDF_MIMETYPE = "application/pdf" @@ -34,6 +34,9 @@ #: Captions and footnotes legitimately overlap their figure, so containment must not drop them. CONTAINMENT_EXEMPT_LABELS = frozenset({DocItemLabel.CAPTION, DocItemLabel.FOOTNOTE}) +#: How far either side of a caption its figure or table may sit, nearest first, before after. +CAPTION_REACH = (-1, 1, -2, 2) + @dataclass(frozen=True) class LayoutBlock: @@ -153,6 +156,29 @@ def sort_body_by_position(doc: DoclingDocument) -> None: doc.body.children.sort(key=lambda ref: _sort_key(doc, ref.resolve(doc))) +def _page_of(item: NodeItem) -> Optional[int]: + provs = getattr(item, "prov", None) or [] + return provs[0].page_no if provs else None + + +def link_captions(doc: DoclingDocument) -> None: + """Attach each unlinked caption to the nearest table or picture on its page, looking back first. + + docling serializes a linked caption inside its owner and nowhere else, so this is what keeps a + caption out of the prose stream and inside the table or figure that it names. + """ + body = [ref.resolve(doc) for ref in doc.body.children] + linked = {ref.cref for item in body if isinstance(item, FloatingItem) for ref in item.captions} + for index, item in enumerate(body): + if getattr(item, "label", None) != DocItemLabel.CAPTION or item.self_ref in linked: + continue + for offset in CAPTION_REACH: + owner = body[index + offset] if 0 <= index + offset < len(body) else None + if isinstance(owner, (TableItem, PictureItem)) and _page_of(owner) == _page_of(item): + owner.captions.append(item.get_ref()) + break + + def append_blocks( doc: DoclingDocument, ctx: PageContext, @@ -185,4 +211,5 @@ def append_blocks( added.append(item) sort_body_by_position(doc) + link_captions(doc) return added diff --git a/extralit-server/src/extralit_server/contexts/ocr/elements.py b/extralit-server/src/extralit_server/contexts/ocr/elements.py index e3dff4117..fc76ef38b 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/elements.py +++ b/extralit-server/src/extralit_server/contexts/ocr/elements.py @@ -5,10 +5,11 @@ — so `contexts/retrieval` never has to know a docling label. Rows in, rows out: chunking re-runs from the Lance dataset without re-parsing the PDF. -The whole projection is one DuckDB statement over the `items` columns. Reading breadcrumbs and -captions out row by row would mean pulling every document's text into Python to build strings -that go straight back into Arrow; as SQL it stays columnar, pushes the projection down to Lance, -and does a whole workspace in the same pass as a single document. +Rendering is docling's, done once at projection time into the `markdown` column. What this adds +is the heading breadcrumb, as window functions over the `items` columns: reading it out row by +row would mean pulling every document's text into Python to build strings that go straight back +into Arrow; as SQL it stays columnar, pushes the projection down to Lance, and does a whole +workspace in the same pass as a single document. """ from __future__ import annotations @@ -34,12 +35,9 @@ #: Headings open a breadcrumb slot. A title sits above every section header, whatever its level. TITLE_SLOT = 0 -#: Both the deepest renderable heading and the deepest breadcrumb slot; `######` has no successor. +#: The deepest breadcrumb slot; anything deeper is nested under the same ancestor anyway. MAX_HEADING_LEVEL = 6 -#: How far either side of a caption its figure or table may sit, nearest first, before after. -CAPTION_REACH = (-1, 1, -2, 2) - ELEMENT_SCHEMA = pa.schema( [ ("document_id", pa.string()), @@ -69,6 +67,24 @@ def _labels(labels: Iterable[DocItemLabel]) -> str: _HEADING_LEVEL = f"least(greatest(coalesce(level, 1), 1), {MAX_HEADING_LEVEL})" +_SLICED = "provs >= 2 AND charspan_start IS NOT NULL AND charspan_end IS NOT NULL AND charspan_end > charspan_start" + +#: A row's share of its item's text; whole when the item sits on one page. +_OWN_TEXT = _STRIP.format( + f"""CASE + WHEN {_SLICED} THEN substr(coalesce(text, ''), charspan_start + 1, charspan_end - charspan_start) + ELSE coalesce(text, '') + END""" +) + +#: Markdown is rendered per item, so a page-spanning item falls back to its share of the raw text. +_CONTENT = _STRIP.format( + f"""CASE + WHEN {_SLICED} AND coalesce(markdown, '') <> '' THEN own_text + ELSE coalesce(markdown, text, '') + END""" +) + def _breadcrumb_columns() -> str: """One text and two positions per slot: a slot is in scope only if nothing has closed it.""" @@ -91,57 +107,12 @@ def _breadcrumb_list() -> str: return f"list_filter(list_value({slots}), heading -> heading IS NOT NULL)" -def _caption_owner() -> str: - """The figure or table a caption binds to: nearest on the same page, looking back first.""" - branches = [] - for offset in CAPTION_REACH: - window, distance = ("lag" if offset < 0 else "lead"), abs(offset) - neighbour = f"{window}({{0}}, {distance}) OVER document" - branches.append( - f"WHEN {neighbour.format('label')} IN {_labels(TABLE_LABELS | PICTURE_LABELS)}" - f" AND {neighbour.format('page_no')} IS NOT DISTINCT FROM page_no" - f" THEN {neighbour.format('ord')}" - ) - return ( - "CASE WHEN label <> 'caption' OR own_text = '' THEN NULL\n " - + "\n ".join(branches) - + "\n END" - ) - - -#: A caption is consumed by its owner, so a table that drops it drops the only prose naming it. -_CAPTION_TAG = ( - "'' || replace(replace(replace(caption, '&', '&'), '<', '<'), '>', '>') || ''" -) - -_TABLE_CONTENT = f"""CASE - WHEN coalesce(html, '') = '' THEN coalesce(caption, '') - WHEN coalesce(caption, '') = '' THEN html - WHEN starts_with(html, '' IN html) > 0 - THEN substr(html, 1, position('>' IN html)) || {_CAPTION_TAG} - || substr(html, position('>' IN html) + 1) - ELSE {_CAPTION_TAG} || html - END""" - -_CONTENT = f"""CASE - WHEN label IN {_labels(TABLE_LABELS)} THEN {_TABLE_CONTENT} - WHEN label IN {_labels(PICTURE_LABELS)} THEN coalesce(caption, '') - WHEN own_text = '' THEN '' - WHEN label = '{DocItemLabel.TITLE.value}' THEN '# ' || own_text - WHEN label = '{DocItemLabel.SECTION_HEADER.value}' - THEN repeat('#', {_HEADING_LEVEL}) || ' ' || own_text - WHEN label = '{DocItemLabel.LIST_ITEM.value}' THEN '- ' || own_text - WHEN label = '{DocItemLabel.CODE.value}' THEN '```' || chr(10) || own_text || chr(10) || '```' - ELSE own_text - END""" - - def elements_sql(source: str = "items") -> str: """The projection, as one statement over anything DuckDB can scan with the `items` columns.""" return f""" WITH ordered AS ( SELECT - document_id, self_ref, level, reading_order, page_no, bbox, html, text, + document_id, self_ref, level, reading_order, page_no, bbox, text, markdown, charspan_start, charspan_end, coalesce(label, '{DocItemLabel.TEXT.value}') AS label, row_number() OVER document AS ord, @@ -150,21 +121,12 @@ def elements_sql(source: str = "items") -> str: WINDOW document AS (PARTITION BY document_id ORDER BY reading_order, prov_index) ), sliced AS ( - SELECT * EXCLUDE (text, charspan_start, charspan_end, provs), - { - _STRIP.format( - '''CASE - WHEN provs >= 2 AND charspan_start IS NOT NULL AND charspan_end IS NOT NULL - AND charspan_end > charspan_start - THEN substr(coalesce(text, ''), charspan_start + 1, charspan_end - charspan_start) - ELSE coalesce(text, '') - END''' - ) - } AS own_text + SELECT *, {_OWN_TEXT} AS own_text FROM ordered ), slotted AS ( - SELECT *, + SELECT * EXCLUDE (text, markdown, charspan_start, charspan_end, provs), + {_CONTENT} AS content, CASE WHEN own_text = '' THEN NULL WHEN label = '{DocItemLabel.TITLE.value}' THEN {TITLE_SLOT} @@ -178,43 +140,21 @@ def elements_sql(source: str = "items") -> str: last_value(slot IGNORE NULLS) OVER running AS heading_level FROM slotted WINDOW running AS ({_RUNNING}) -), -neighbours AS ( - SELECT * EXCLUDE (slot), - {_breadcrumb_list()} AS headings, - {_caption_owner()} AS owner_ord - FROM breadcrumbs - WINDOW document AS (PARTITION BY document_id ORDER BY ord) -), -captions AS ( - SELECT document_id, owner_ord, string_agg(own_text, ' ' ORDER BY ord) AS caption - FROM neighbours - WHERE owner_ord IS NOT NULL - GROUP BY document_id, owner_ord -), -projected AS ( - SELECT - element.document_id, - element.ord, - CASE - WHEN label IN {_labels(TABLE_LABELS)} THEN '{TABLE}' - WHEN label IN {_labels(PICTURE_LABELS)} THEN '{FIGURE}' - ELSE '{MARKDOWN}' - END AS type, - {_CONTENT} AS content, - element.page_no, element.bbox, element.label, element.level, - element.self_ref AS item_ref, element.reading_order, - element.headings, element.heading_level - FROM neighbours AS element - LEFT JOIN captions - ON captions.document_id = element.document_id AND captions.owner_ord = element.ord - WHERE label NOT IN {_labels(SKIPPED_LABELS)} - AND element.owner_ord IS NULL ) -SELECT document_id, type, content, page_no, bbox, label, level, item_ref, reading_order, - headings, heading_level -FROM projected -WHERE content <> '' +SELECT + document_id, + CASE + WHEN label IN {_labels(TABLE_LABELS)} THEN '{TABLE}' + WHEN label IN {_labels(PICTURE_LABELS)} THEN '{FIGURE}' + ELSE '{MARKDOWN}' + END AS type, + content, page_no, bbox, label, level, + self_ref AS item_ref, reading_order, + {_breadcrumb_list()} AS headings, + heading_level +FROM breadcrumbs +WHERE label NOT IN {_labels(SKIPPED_LABELS)} + AND content <> '' ORDER BY document_id, ord """ diff --git a/extralit-server/src/extralit_server/contexts/ocr/layout_store.py b/extralit-server/src/extralit_server/contexts/ocr/layout_store.py index c2b2f96e2..6396035b9 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/layout_store.py +++ b/extralit-server/src/extralit_server/contexts/ocr/layout_store.py @@ -44,6 +44,18 @@ _SCHEMAS = {ITEMS_DATASET: ITEM_SCHEMA, PAGES_DATASET: PAGE_SCHEMA} +# Lance's SQL type names for `add_columns`; anything else needs a real migration, not a NULL fill. +_SQL_TYPES = { + pa.string(): "string", + pa.bool_(): "boolean", + pa.int8(): "tinyint", + pa.int16(): "smallint", + pa.int32(): "int", + pa.int64(): "bigint", + pa.float32(): "float", + pa.float64(): "double", +} + def layout_root(workspace_name: str) -> tuple[str, Optional[dict[str, str]]]: """Root of the workspace's datasets, resolved exactly like every other artifact of it.""" @@ -138,6 +150,14 @@ def open(self, name: str) -> Optional[lance.LanceDataset]: def _write(self, name: str, data: pa.Table, mode: str) -> int: return lance.write_dataset(data, self.uri(name), mode=mode, storage_options=self.storage_options).version + def _add_missing_columns(self, dataset: lance.LanceDataset, schema: pa.Schema) -> lance.LanceDataset: + """Widen a dataset written under an older schema; new columns are NULL for the rows already there.""" + missing = [field for field in schema if field.name not in dataset.schema.names] + if not missing: + return dataset + dataset.add_columns({field.name: f"CAST(NULL AS {_SQL_TYPES[field.type]})" for field in missing}) + return lance.dataset(dataset.uri, storage_options=self.storage_options) + def _replace_one(self, name: str, document_id: UUID | str, data: pa.Table) -> int: dataset = self.open(name) if dataset is None: @@ -147,6 +167,7 @@ def _replace_one(self, name: str, document_id: UUID | str, data: pa.Table) -> in # Another worker created it between the open and the write; join it instead. dataset = lance.dataset(self.uri(name), storage_options=self.storage_options) + dataset = self._add_missing_columns(dataset, data.schema) dataset.delete(_document_filter(document_id)) if data.num_rows == 0: return lance.dataset(self.uri(name), storage_options=self.storage_options).version diff --git a/extralit-server/tests/unit/contexts/ocr/test_arrow.py b/extralit-server/tests/unit/contexts/ocr/test_arrow.py index 71e8c431e..286e7d46c 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_arrow.py +++ b/extralit-server/tests/unit/contexts/ocr/test_arrow.py @@ -145,6 +145,35 @@ def test_text_items_have_no_html(self, doc): assert rows["#/texts/2"]["html"] is None + def test_markdown_is_docling_rendering_of_each_item(self, doc): + rows = {r["self_ref"]: r for r in items_table(doc, DOCUMENT_ID).to_pylist()} + + assert rows["#/texts/0"]["markdown"] == "# A Paper" + assert rows["#/texts/1"]["markdown"] == "### Methods" + assert rows["#/texts/2"]["markdown"] == "Body text here." + assert rows["#/tables/0"]["markdown"].startswith("| Group") + assert rows["#/pictures/0"]["markdown"] == "" + + def test_markdown_keeps_the_characters_a_search_index_needs(self): + document = new_document("sample") + ctx = PageContext(page_no=1, size=Size(width=612, height=792)) + append_blocks( + document, ctx, [LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text="p_value < 0.05 & n_obs")] + ) + + rows = items_table(document, DOCUMENT_ID).to_pylist() + + assert rows[0]["markdown"] == "p_value < 0.05 & n_obs" + + def test_markdown_is_rendered_once_per_item_not_once_per_provenance(self, doc): + for item in doc.texts: + if item.self_ref == "#/texts/2": + item.prov.append(ProvenanceItem(page_no=2, bbox=bbox(t=10, b=30), charspan=(0, 0))) + + rows = [r for r in items_table(doc, DOCUMENT_ID).to_pylist() if r["self_ref"] == "#/texts/2"] + + assert [r["markdown"] for r in rows] == ["Body text here."] * 2 + def test_parent_ref_is_recorded(self, doc): rows = {r["self_ref"]: r for r in items_table(doc, DOCUMENT_ID).to_pylist()} diff --git a/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py b/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py index 2de355d5c..22e9e1721 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py +++ b/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py @@ -213,6 +213,80 @@ def test_captions_are_kept_even_when_they_touch_a_figure(self, doc, ctx): assert [t.text for t in doc.texts] == ["Figure 1."] + def test_a_caption_is_linked_to_the_figure_it_touches(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=100, b=300, left=0, right=500)), + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=310, b=320), text="Figure 1."), + ] + + append_blocks(doc, ctx, blocks) + + assert doc.pictures[0].caption_text(doc) == "Figure 1." + + def test_a_caption_above_its_table_binds_to_it(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=90, b=99), text="Table 1."), + LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=300)), + ] + + append_blocks(doc, ctx, blocks) + + assert doc.tables[0].caption_text(doc) == "Table 1." + + def test_the_nearer_neighbour_wins_when_a_caption_sits_between_two_figures(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=100, b=200)), + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=210, b=220), text="Belongs to the first."), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=230, b=240), text="Prose."), + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=300, b=400)), + ] + + append_blocks(doc, ctx, blocks) + + assert doc.pictures[0].caption_text(doc) == "Belongs to the first." + assert doc.pictures[1].captions == [] + + def test_a_caption_reaches_past_one_line_of_prose(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=100, b=200)), + LayoutBlock(label=DocItemLabel.FOOTNOTE, bbox=bbox(t=205, b=208), text="a"), + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=210, b=220), text="Figure 1."), + ] + + append_blocks(doc, ctx, blocks) + + assert doc.pictures[0].caption_text(doc) == "Figure 1." + + def test_two_captions_on_one_figure_are_both_linked(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=90, b=99), text="Figure 1."), + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=100, b=200)), + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=210, b=220), text="A red square."), + ] + + append_blocks(doc, ctx, blocks) + + assert [ref.resolve(doc).text for ref in doc.pictures[0].captions] == ["Figure 1.", "A red square."] + + def test_a_caption_never_binds_across_a_page(self, doc, ctx): + append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=700, b=780))]) + page_two = PageContext(page_no=2, size=ctx.size) + + append_blocks(doc, page_two, [LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=10, b=20), text="Orphan.")]) + + assert doc.pictures[0].captions == [] + + def test_a_second_pass_over_a_page_links_nothing_twice(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=100, b=200)), + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=210, b=220), text="Figure 1."), + ] + append_blocks(doc, ctx, blocks) + + append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=400, b=420), text="More.")]) + + assert len(doc.pictures[0].captions) == 1 + def test_every_item_carries_a_full_provenance_triple(self, doc, ctx): blocks = [ LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text="hello"), diff --git a/extralit-server/tests/unit/contexts/ocr/test_elements.py b/extralit-server/tests/unit/contexts/ocr/test_elements.py index c7258396a..90734459c 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_elements.py +++ b/extralit-server/tests/unit/contexts/ocr/test_elements.py @@ -4,7 +4,7 @@ import pytest from docling_core.types.doc import BoundingBox, CoordOrigin, DocItemLabel, Size -from extralit_server.contexts.ocr.arrow import ITEM_SCHEMA, item_rows, table_html +from extralit_server.contexts.ocr.arrow import ITEM_SCHEMA, item_rows from extralit_server.contexts.ocr.docling_builder import ( LayoutBlock, PageContext, @@ -27,7 +27,8 @@ def bbox(t: float, b: float, left: float = 10.0, right: float = 100.0) -> Boundi return BoundingBox(l=left, t=t, r=right, b=b, coord_origin=CoordOrigin.TOPLEFT) -def row(label, reading_order, **overrides): +def row(label, reading_order, text=None, markdown=None, **overrides): + """An `items` row. `markdown` defaults to `text`, the way docling renders plain prose.""" base = { "document_id": DOCUMENT_ID, "self_ref": f"#/texts/{reading_order}", @@ -40,7 +41,8 @@ def row(label, reading_order, **overrides): "page_no": 1, "bbox": [0.0, 0.0, 10.0, 10.0], "coord_origin": "TOPLEFT", - "text": None, + "text": text, + "markdown": text if markdown is None else markdown, "html": None, "charspan_start": None, "charspan_end": None, @@ -89,10 +91,18 @@ def test_breadcrumb_deepens_and_unwinds_with_heading_level(self): ] def test_a_heading_carries_itself_so_a_chunk_knows_its_own_path(self): - rows = [row(DocItemLabel.SECTION_HEADER, 0, level=1, text="Methods")] + rows = [row(DocItemLabel.SECTION_HEADER, 0, level=1, text="Methods", markdown="## Methods")] assert elements(rows)[0]["headings"] == ["Methods"] + def test_the_breadcrumb_is_the_raw_heading_text_not_its_markdown(self): + rows = [ + row(DocItemLabel.SECTION_HEADER, 0, level=1, text="Methods", markdown="## Methods"), + row(DocItemLabel.TEXT, 1, text="Body."), + ] + + assert [e["headings"] for e in elements(rows)] == [["Methods"], ["Methods"]] + def test_heading_level_tracks_the_deepest_slot_in_scope(self): rows = [ row(DocItemLabel.TITLE, 0, text="A Paper"), @@ -109,15 +119,13 @@ def test_prose_before_any_heading_has_no_breadcrumb(self): def test_a_blank_heading_opens_no_slot(self): rows = [ - row(DocItemLabel.SECTION_HEADER, 0, level=1, text=" "), + row(DocItemLabel.SECTION_HEADER, 0, level=1, text=" ", markdown=""), row(DocItemLabel.TEXT, 1, text="Body."), ] assert [e["headings"] for e in elements(rows)] == [[]] def test_levels_past_h6_share_the_deepest_slot(self): - # Both render as `######`, so nesting one under the other would be a distinction - # the markdown cannot carry. rows = [ row(DocItemLabel.SECTION_HEADER, 0, level=7, text="Seven"), row(DocItemLabel.SECTION_HEADER, 1, level=99, text="Ninety-nine"), @@ -126,122 +134,41 @@ def test_levels_past_h6_share_the_deepest_slot(self): assert [e["headings"] for e in elements(rows)] == [["Seven"], ["Ninety-nine"]] -class TestMarkdownRendering: - @pytest.mark.parametrize( - "label, level, text, expected", - [ - (DocItemLabel.TITLE, None, "A Paper", "# A Paper"), - (DocItemLabel.SECTION_HEADER, 3, "Deep", "### Deep"), - (DocItemLabel.SECTION_HEADER, None, "Unlevelled", "# Unlevelled"), - (DocItemLabel.LIST_ITEM, None, "first", "- first"), - (DocItemLabel.TEXT, None, "Prose.", "Prose."), - (DocItemLabel.CODE, None, "x = 1", "```\nx = 1\n```"), - ], - ) - def test_rows_render_as_the_markdown_the_chunker_rules_expect(self, label, level, text, expected): - element = elements([row(label, 0, level=level, text=text)])[0] +class TestContent: + def test_content_is_the_rendered_markdown_not_the_raw_text(self): + rows = [row(DocItemLabel.SECTION_HEADER, 0, level=1, text="Methods", markdown="## Methods")] - assert element["type"] == MARKDOWN - assert element["content"] == expected + element = elements(rows)[0] - def test_heading_level_is_clamped_to_six(self): - assert elements([row(DocItemLabel.SECTION_HEADER, 0, level=99, text="Deep")])[0]["content"] == "###### Deep" + assert element["type"] == MARKDOWN + assert element["content"] == "## Methods" def test_surrounding_whitespace_is_stripped_including_newlines(self): assert elements([row(DocItemLabel.TEXT, 0, text=" \n\tProse.\n ")])[0]["content"] == "Prose." + def test_a_row_rendered_blank_is_dropped(self): + # A linked caption renders as '' on its own: docling folds it into the table or figure. + assert elements([row(DocItemLabel.CAPTION, 0, text="Figure 1.", markdown="")]) == [] -class TestCaptions: - def test_a_caption_is_absorbed_by_its_figure_rather_than_left_as_prose(self): - rows = [ - row(DocItemLabel.PICTURE, 0), - row(DocItemLabel.CAPTION, 1, text="Figure 1. A red square."), - ] - - assert [(e["type"], e["content"]) for e in elements(rows)] == [(FIGURE, "Figure 1. A red square.")] - - def test_a_caption_preceding_its_table_is_absorbed_and_kept(self): - rows = [ - row(DocItemLabel.CAPTION, 0, text="Table 1. Counts."), - row(DocItemLabel.TABLE, 1, html="
x
"), - ] - - found = elements(rows) - - assert [e["type"] for e in found] == [TABLE] - # Consumed from the markdown stream, so the table has to carry it or it is lost. - assert found[0]["content"] == ( - "
Table 1. Counts.
x
" - ) - - def test_a_table_caption_lands_where_html_allows_a_caption(self): - rows = [ - row(DocItemLabel.TABLE, 0, html="
N
"), - row(DocItemLabel.CAPTION, 1, text="Table 2. Sizes."), - ] + def test_a_row_never_rendered_falls_back_to_its_text(self): + # NULL is a row written before the column existed, or one the renderer could not handle. + assert elements([row(DocItemLabel.TEXT, 0, text="Old prose.", markdown="")]) == [] + rows = [{**row(DocItemLabel.TEXT, 0, text="Old prose."), "markdown": None}] - # is only valid as the first child of . - assert elements(rows)[0]["content"].startswith("
") + assert [e["content"] for e in elements(rows)] == ["Old prose."] - def test_a_table_caption_is_escaped(self): + def test_tables_and_figures_carry_their_rendering_whatever_their_text(self): rows = [ - row(DocItemLabel.TABLE, 0, html="
Table 2. Sizes.
x
"), - row(DocItemLabel.CAPTION, 1, text="Risk & spread "), + row(DocItemLabel.TABLE, 0, markdown="Table 1.\n\n| N |\n|---|\n| 1 |"), + row(DocItemLabel.PICTURE, 1, markdown="Figure 1."), + row(DocItemLabel.PICTURE, 2, markdown=""), ] - assert "Risk & spread <n=42>" in elements(rows)[0]["content"] - - def test_a_caption_survives_a_table_that_produced_no_markup(self): - rows = [ - row(DocItemLabel.TABLE, 0, html=None), - row(DocItemLabel.CAPTION, 1, text="Table 3. Unparsed."), + assert [(e["type"], e["content"]) for e in elements(rows)] == [ + (TABLE, "Table 1.\n\n| N |\n|---|\n| 1 |"), + (FIGURE, "Figure 1."), ] - assert [(e["type"], e["content"]) for e in elements(rows)] == [(TABLE, "Table 3. Unparsed.")] - - def test_a_caption_keeps_markup_that_is_not_a_table_element(self): - rows = [ - row(DocItemLabel.TABLE, 0, html="not-a-table"), - row(DocItemLabel.CAPTION, 1, text="Table 4. Odd."), - ] - - assert elements(rows)[0]["content"] == "Table 4. Odd.not-a-table" - - def test_a_caption_on_a_page_with_no_figure_survives_as_prose(self): - rows = [ - row(DocItemLabel.PICTURE, 0, page_no=2), - row(DocItemLabel.CAPTION, 1, page_no=1, text="Orphaned."), - ] - - assert [(e["type"], e["content"]) for e in elements(rows)] == [(MARKDOWN, "Orphaned.")] - - def test_the_nearer_neighbour_wins_when_a_caption_sits_between_two_figures(self): - rows = [ - row(DocItemLabel.PICTURE, 0), - row(DocItemLabel.CAPTION, 1, text="Belongs to the first."), - row(DocItemLabel.TEXT, 2, text="Prose."), - row(DocItemLabel.PICTURE, 3), - ] - - found = elements(rows) - - assert [(e["type"], e["content"]) for e in found] == [ - (FIGURE, "Belongs to the first."), - (MARKDOWN, "Prose."), - ] - - def test_two_captions_on_one_figure_are_joined(self): - rows = [ - row(DocItemLabel.CAPTION, 0, text="Figure 1."), - row(DocItemLabel.PICTURE, 1), - row(DocItemLabel.CAPTION, 2, text="A red square."), - ] - - assert [(e["type"], e["content"]) for e in elements(rows)] == [(FIGURE, "Figure 1. A red square.")] - - def test_an_uncaptioned_figure_yields_nothing_retrievable(self): - assert elements([row(DocItemLabel.PICTURE, 0)]) == [] - class TestProvenance: def test_running_headers_and_footers_are_dropped(self): @@ -254,20 +181,49 @@ def test_running_headers_and_footers_are_dropped(self): assert [e["content"] for e in elements(rows)] == ["Real body."] def test_one_element_per_provenance_row_when_an_item_spans_a_page_break(self): + text = "first half second half" spanning = [ - row(DocItemLabel.TEXT, 0, page_no=1, text="first half second half", charspan_start=0, charspan_end=11), + row(DocItemLabel.TEXT, 0, page_no=1, text=text, charspan_start=0, charspan_end=11), + row(DocItemLabel.TEXT, 0, page_no=2, prov_index=1, text=text, charspan_start=11, charspan_end=22), + ] + + assert [(e["page_no"], e["content"]) for e in elements(spanning)] == [(1, "first half"), (2, "second half")] + + def test_a_page_spanning_heading_keeps_its_slice_and_loses_its_marker(self): + # Markdown is rendered per item and cannot be sliced by charspan, so the raw slice wins. + text = "Methods and Materials" + spanning = [ + row( + DocItemLabel.SECTION_HEADER, + 0, + level=1, + text=text, + markdown=f"## {text}", + charspan_start=0, + charspan_end=7, + ), row( - DocItemLabel.TEXT, + DocItemLabel.SECTION_HEADER, 0, + level=1, page_no=2, prov_index=1, - text="first half second half", - charspan_start=11, - charspan_end=22, + text=text, + markdown=f"## {text}", + charspan_start=7, + charspan_end=21, ), ] - assert [(e["page_no"], e["content"]) for e in elements(spanning)] == [(1, "first half"), (2, "second half")] + assert [e["content"] for e in elements(spanning)] == ["Methods", "and Materials"] + + def test_a_page_spanning_table_repeats_its_rendering_on_both_pages(self): + spanning = [ + row(DocItemLabel.TABLE, 0, page_no=1, markdown="| N |", charspan_start=0, charspan_end=0), + row(DocItemLabel.TABLE, 0, page_no=2, prov_index=1, markdown="| N |", charspan_start=0, charspan_end=0), + ] + + assert [(e["page_no"], e["content"]) for e in elements(spanning)] == [(1, "| N |"), (2, "| N |")] def test_elements_come_back_in_reading_order_whatever_the_row_order(self): rows = [row(DocItemLabel.TEXT, 2, text="third"), row(DocItemLabel.TEXT, 0, text="first")] @@ -300,106 +256,75 @@ def test_documents_are_projected_in_one_pass_without_bleeding_into_each_other(se ("bbbb", ["bbbb heading"]), ] - def test_a_caption_never_binds_across_a_document_boundary(self): - rows = [ - {**row(DocItemLabel.PICTURE, 0), "document_id": "aaaa"}, - {**row(DocItemLabel.CAPTION, 0, text="Belongs to bbbb."), "document_id": "bbbb"}, - ] - - # The figure stays uncaptioned and drops out; the caption stays prose in its own document. - assert [(e["document_id"], e["type"], e["content"]) for e in elements(rows)] == [ - ("bbbb", MARKDOWN, "Belongs to bbbb.") - ] - -class TestTableHtml: +class TestAgainstRealProjection: @pytest.fixture - def table(self): + def found(self): document = new_document("sample") append_blocks( document, PageContext(page_no=1, size=Size(width=612, height=792)), [ + LayoutBlock(label=DocItemLabel.TITLE, bbox=bbox(t=1, b=5), text="A Paper"), + LayoutBlock(label=DocItemLabel.SECTION_HEADER, bbox=bbox(t=10, b=40), text="Methods", level=1), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=50, b=70), text="Body text."), + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=190, b=199), text="Table 1. Counts."), LayoutBlock( label=DocItemLabel.TABLE, bbox=bbox(t=200, b=400), - cells=[ - make_cell("Group", row=0, col=0, column_header=True), - make_cell("N", row=0, col=1, column_header=True), - make_cell("control", row=1, col=0), - make_cell("4 < 2", row=1, col=1), - ], - ) + cells=[make_cell("N", row=0, col=0, column_header=True), make_cell("42", row=1, col=0)], + ), + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=500, b=600)), + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=601, b=610), text="Figure 1. A red square."), + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=700, b=750)), ], ) - return document.tables[0] - - def test_header_cells_land_in_thead_not_tbody(self, table): - assert table_html(table) == ( - "" - "" - "" - "
GroupN
control4 < 2
" - ) + return elements(item_rows(document, DOCUMENT_ID)) - def test_docling_own_exporter_would_have_buried_the_header(self, table): - # The reason this serializer exists: a row-window chunk cannot repeat a header it - # cannot find, and docling puts the cells inside . - assert "" not in table.export_to_html() + def test_captions_ride_inside_their_table_and_figure_not_the_prose(self, found): + assert [e["type"] for e in found] == [MARKDOWN, MARKDOWN, MARKDOWN, TABLE, FIGURE] + assert found[3]["content"].startswith("Table 1. Counts.\n\n|") + assert "|-----|" in found[3]["content"] and "42" in found[3]["content"] + assert found[4]["content"] == "Figure 1. A red square." - def test_spans_are_carried_as_attributes(self): - document = new_document("sample") - append_blocks( - document, - PageContext(page_no=1, size=Size(width=612, height=792)), - [ - LayoutBlock( - label=DocItemLabel.TABLE, - bbox=bbox(t=200, b=400), - cells=[ - make_cell("Both", row=0, col=0, col_span=2, column_header=True), - make_cell("tall", row=1, col=0, row_span=2), - make_cell("x", row=1, col=1), - ], - ) - ], - ) + def test_an_uncaptioned_figure_yields_nothing_retrievable(self, found): + assert sum(e["type"] == FIGURE for e in found) == 1 - html = table_html(document.tables[0]) + def test_docling_reserves_the_top_heading_for_the_title(self, found): + assert [e["content"] for e in found[:2]] == ["# A Paper", "## Methods"] - assert 'Both' in html - assert 'tall' in html + def test_the_breadcrumb_reaches_every_element(self, found): + assert all(e["headings"] == ["A Paper", "Methods"] for e in found[1:]) - def test_a_table_with_no_cells_has_no_html(self): - document = new_document("sample") - append_blocks( - document, - PageContext(page_no=1, size=Size(width=612, height=792)), - [LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=200, b=400), cells=[])], - ) - assert table_html(document.tables[0]) is None +class TestTableChunking: + """chonkie owns the row-window split; pin the behaviour the element design leans on.""" + def test_a_table_split_into_row_windows_repeats_caption_and_header_on_every_chunk(self): + from chonkie import TableChunker -class TestAgainstRealProjection: - def test_elements_line_up_with_a_projected_document(self): document = new_document("sample") append_blocks( document, PageContext(page_no=1, size=Size(width=612, height=792)), [ - LayoutBlock(label=DocItemLabel.SECTION_HEADER, bbox=bbox(t=10, b=40), text="Methods", level=1), - LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=50, b=70), text="Body text."), + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=190, b=199), text="Table 1. Counts."), LayoutBlock( label=DocItemLabel.TABLE, bbox=bbox(t=200, b=400), - cells=[make_cell("N", row=0, col=0, column_header=True), make_cell("42", row=1, col=0)], + cells=[ + make_cell("Group", row=0, col=0, column_header=True), + make_cell("N", row=0, col=1, column_header=True), + *[make_cell(f"g{i}", row=i, col=0) for i in range(1, 7)], + *[make_cell(str(i), row=i, col=1) for i in range(1, 7)], + ], ), ], ) + content = elements(item_rows(document, DOCUMENT_ID))[0]["content"] - found = elements(item_rows(document, DOCUMENT_ID)) + chunks = [chunk.text for chunk in TableChunker(chunk_size=2)(content)] - assert [e["type"] for e in found] == [MARKDOWN, MARKDOWN, TABLE] - assert all(e["headings"] == ["Methods"] for e in found) - assert "" in found[-1]["content"] + assert len(chunks) > 1 + assert all("Table 1. Counts." in chunk and "| Group" in chunk for chunk in chunks) + assert "| g6" in chunks[-1] and "| g6" not in chunks[0] diff --git a/extralit-server/tests/unit/contexts/ocr/test_layout_store.py b/extralit-server/tests/unit/contexts/ocr/test_layout_store.py index 840870c86..846c99615 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_layout_store.py +++ b/extralit-server/tests/unit/contexts/ocr/test_layout_store.py @@ -6,6 +6,7 @@ import time from uuid import uuid4 +import lance import pyarrow as pa import pytest @@ -45,6 +46,7 @@ def items(document_id: str, count: int = 3, label: str = "text") -> pa.Table: "charspan_start": 0, "charspan_end": 5, "text": f"row {i}", + "markdown": f"row {i}", "html": None, } for i in range(count) @@ -129,6 +131,17 @@ def test_a_zero_row_document_still_clears_its_old_rows(self, local_store): assert local_store.load_items(document_id).num_rows == 0 assert local_store.load_pages(document_id).num_rows == 0 + def test_a_dataset_written_under_an_older_schema_is_widened_not_rejected(self, local_store): + old, new = str(uuid4()), str(uuid4()) + narrow = items(old, 2).drop_columns(["markdown"]) + lance.write_dataset(narrow, local_store.items_uri(), mode="create") + + local_store.replace_document(new, items(new, 1), pages(new)) + + assert set(local_store.open(ITEMS_DATASET).schema.names) == set(ITEM_SCHEMA.names) + assert local_store.load_items(old, columns=["markdown"]).to_pylist() == [{"markdown": None}] * 2 + assert local_store.load_items(new, columns=["markdown"]).to_pylist() == [{"markdown": "row 0"}] + def test_replace_reports_the_dataset_versions(self, local_store): document_id = str(uuid4()) From cb55a37330df013dd0673721f754a6d4a3d0b652 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Fri, 4 Sep 2026 00:21:25 -0700 Subject: [PATCH 11/16] refactor(ocr): collapse the element projection to two CTEs Page-spanning rows are detected from the charspan alone (short of the whole text), which drops the per-item count window; own_text, content and slot are lateral aliases in one SELECT; the breadcrumb keeps two columns per slot (latest heading, depth of the latest heading at or above) instead of three, and heading_level is the deepest slot's depth. Single-consumer helpers folded into their callers. --- .../src/extralit_server/contexts/ocr/arrow.py | 17 +-- .../extralit_server/contexts/ocr/elements.py | 123 ++++++------------ 2 files changed, 47 insertions(+), 93 deletions(-) diff --git a/extralit-server/src/extralit_server/contexts/ocr/arrow.py b/extralit-server/src/extralit_server/contexts/ocr/arrow.py index d8412748e..38fd5eba6 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/arrow.py +++ b/extralit-server/src/extralit_server/contexts/ocr/arrow.py @@ -60,15 +60,7 @@ def _table_html(doc: DoclingDocument, item: DocItem) -> Optional[str]: return None -def markdown_serializer(doc: DoclingDocument) -> MarkdownDocSerializer: - """docling's renderer, tuned for retrieval: raw characters, no image placeholder.""" - return MarkdownDocSerializer( - doc=doc, - params=MarkdownParams(escape_html=False, escape_underscores=False, image_placeholder=""), - ) - - -def _item_markdown(serializer: MarkdownDocSerializer, item: DocItem) -> Optional[str]: +def _serialize(serializer: MarkdownDocSerializer, item: DocItem) -> Optional[str]: try: return serializer.serialize(item=item).text except Exception: # NULL, not '': a row the renderer could not handle is not a blank one @@ -78,7 +70,10 @@ def _item_markdown(serializer: MarkdownDocSerializer, item: DocItem) -> Optional def item_rows(doc: DoclingDocument, document_id: str) -> list[dict[str, Any]]: """Flatten every item's provenance into rows matching `ITEM_SCHEMA`.""" rows: list[dict[str, Any]] = [] - serializer = markdown_serializer(doc) + # Raw characters and no image placeholder: this column feeds a search index, not a reader. + markdown = MarkdownDocSerializer( + doc=doc, params=MarkdownParams(escape_html=False, escape_underscores=False, image_placeholder="") + ) for reading_order, (item, _level) in enumerate(doc.iterate_items(with_groups=False)): base = { @@ -90,7 +85,7 @@ def item_rows(doc: DoclingDocument, document_id: str) -> list[dict[str, Any]]: "level": getattr(item, "level", None), "reading_order": reading_order, "text": getattr(item, "text", None) or None, - "markdown": _item_markdown(serializer, item), + "markdown": _serialize(markdown, item), "html": _table_html(doc, item), } diff --git a/extralit-server/src/extralit_server/contexts/ocr/elements.py b/extralit-server/src/extralit_server/contexts/ocr/elements.py index fc76ef38b..b62c5b79d 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/elements.py +++ b/extralit-server/src/extralit_server/contexts/ocr/elements.py @@ -6,15 +6,12 @@ from the Lance dataset without re-parsing the PDF. Rendering is docling's, done once at projection time into the `markdown` column. What this adds -is the heading breadcrumb, as window functions over the `items` columns: reading it out row by -row would mean pulling every document's text into Python to build strings that go straight back -into Arrow; as SQL it stays columnar, pushes the projection down to Lance, and does a whole -workspace in the same pass as a single document. +is the heading breadcrumb, as window functions over the `items` columns, so a whole workspace is +projected in the same columnar pass as a single document. """ from __future__ import annotations -from collections.abc import Iterable from typing import TYPE_CHECKING, Any, Optional import pyarrow as pa @@ -55,91 +52,55 @@ ) -def _labels(labels: Iterable[DocItemLabel]) -> str: - """Render a label set as a SQL `IN` list. Enum values only — nothing here is caller input.""" +def _labels(labels: frozenset[DocItemLabel]) -> str: + """A label set as a SQL `IN` list. Enum values only — nothing here is caller input.""" return "(" + ", ".join(f"'{label.value}'" for label in sorted(labels)) + ")" -#: Python's `str.strip`; DuckDB's bare `trim` takes spaces off and leaves newlines behind. -_STRIP = r"regexp_replace({0}, '^\s+|\s+$', '', 'g')" - -_RUNNING = "PARTITION BY document_id ORDER BY ord ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW" - -_HEADING_LEVEL = f"least(greatest(coalesce(level, 1), 1), {MAX_HEADING_LEVEL})" - -_SLICED = "provs >= 2 AND charspan_start IS NOT NULL AND charspan_end IS NOT NULL AND charspan_end > charspan_start" - -#: A row's share of its item's text; whole when the item sits on one page. -_OWN_TEXT = _STRIP.format( - f"""CASE - WHEN {_SLICED} THEN substr(coalesce(text, ''), charspan_start + 1, charspan_end - charspan_start) - ELSE coalesce(text, '') - END""" -) - -#: Markdown is rendered per item, so a page-spanning item falls back to its share of the raw text. -_CONTENT = _STRIP.format( - f"""CASE - WHEN {_SLICED} AND coalesce(markdown, '') <> '' THEN own_text - ELSE coalesce(markdown, text, '') - END""" -) - - -def _breadcrumb_columns() -> str: - """One text and two positions per slot: a slot is in scope only if nothing has closed it.""" - parts = [] - for slot in range(TITLE_SLOT, MAX_HEADING_LEVEL + 1): - parts += [ - f"last_value(CASE WHEN slot = {slot} THEN own_text END IGNORE NULLS) OVER running AS heading_{slot}", - f"max(CASE WHEN slot = {slot} THEN ord END) OVER running AS opened_{slot}", - f"max(CASE WHEN slot <= {slot} THEN ord END) OVER running AS covered_{slot}", - ] - return ",\n ".join(parts) - - -def _breadcrumb_list() -> str: - """A slot survives only while the newest heading at or above it is still its own.""" - slots = ", ".join( - f"CASE WHEN opened_{slot} = covered_{slot} THEN heading_{slot} END" - for slot in range(TITLE_SLOT, MAX_HEADING_LEVEL + 1) - ) - return f"list_filter(list_value({slots}), heading -> heading IS NOT NULL)" - - def elements_sql(source: str = "items") -> str: """The projection, as one statement over anything DuckDB can scan with the `items` columns.""" + slots = range(TITLE_SLOT, MAX_HEADING_LEVEL + 1) + # Per slot: the latest heading opened there, and the depth of the latest heading at or above + # it. The slot is still in scope only while those coincide. + breadcrumb = ",\n ".join( + f"last_value(CASE WHEN slot = {s} THEN own_text END IGNORE NULLS) OVER running AS heading_{s},\n " + f"last_value(CASE WHEN slot <= {s} THEN slot END IGNORE NULLS) OVER running AS newest_{s}" + for s in slots + ) + headings = ", ".join(f"CASE WHEN newest_{s} = {s} THEN heading_{s} END" for s in slots) + # Python's `str.strip`; DuckDB's bare `trim` takes spaces off and leaves newlines behind. + strip = r"regexp_replace({}, '^\s+|\s+$', '', 'g')" + return f""" -WITH ordered AS ( - SELECT - document_id, self_ref, level, reading_order, page_no, bbox, text, markdown, - charspan_start, charspan_end, +WITH sliced AS ( + SELECT * EXCLUDE (label), coalesce(label, '{DocItemLabel.TEXT.value}') AS label, - row_number() OVER document AS ord, - count(*) OVER (PARTITION BY document_id, self_ref) AS provs - FROM {source} - WINDOW document AS (PARTITION BY document_id ORDER BY reading_order, prov_index) -), -sliced AS ( - SELECT *, {_OWN_TEXT} AS own_text - FROM ordered -), -slotted AS ( - SELECT * EXCLUDE (text, markdown, charspan_start, charspan_end, provs), - {_CONTENT} AS content, + row_number() OVER (PARTITION BY document_id ORDER BY reading_order, prov_index) AS ord, + -- A charspan short of the whole text is one page's share of an item that spans a break. + coalesce(charspan_start > 0 OR charspan_end < length(text), false) AS partial, + {strip.format("CASE WHEN partial THEN text[charspan_start + 1 : charspan_end] ELSE coalesce(text, '') END")} + AS own_text, + -- Markdown is rendered per item, so a partial row falls back to its share of the raw text. + CASE + WHEN markdown = '' THEN '' + WHEN partial OR markdown IS NULL THEN own_text + ELSE {strip.format("markdown")} + END AS content, CASE WHEN own_text = '' THEN NULL WHEN label = '{DocItemLabel.TITLE.value}' THEN {TITLE_SLOT} - WHEN label = '{DocItemLabel.SECTION_HEADER.value}' THEN {_HEADING_LEVEL} + WHEN label = '{DocItemLabel.SECTION_HEADER.value}' + THEN least(greatest(coalesce(level, 1), 1), {MAX_HEADING_LEVEL}) END AS slot - FROM sliced + FROM {source} ), breadcrumbs AS ( SELECT *, - {_breadcrumb_columns()}, - last_value(slot IGNORE NULLS) OVER running AS heading_level - FROM slotted - WINDOW running AS ({_RUNNING}) + {breadcrumb} + FROM sliced + WINDOW running AS ( + PARTITION BY document_id ORDER BY ord ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ) ) SELECT document_id, @@ -148,13 +109,11 @@ def elements_sql(source: str = "items") -> str: WHEN label IN {_labels(PICTURE_LABELS)} THEN '{FIGURE}' ELSE '{MARKDOWN}' END AS type, - content, page_no, bbox, label, level, - self_ref AS item_ref, reading_order, - {_breadcrumb_list()} AS headings, - heading_level + content, page_no, bbox, label, level, self_ref AS item_ref, reading_order, + list_filter([{headings}], heading -> heading IS NOT NULL) AS headings, + newest_{MAX_HEADING_LEVEL} AS heading_level FROM breadcrumbs -WHERE label NOT IN {_labels(SKIPPED_LABELS)} - AND content <> '' +WHERE label NOT IN {_labels(SKIPPED_LABELS)} AND content <> '' ORDER BY document_id, ord """ From c9648d4c50a3fed0abdaa100501cd581a9a444f8 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Fri, 4 Sep 2026 00:35:11 -0700 Subject: [PATCH 12/16] docs: rule against reimplementing what the libraries already do --- CLAUDE.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 3cbfc6d96..4ffdedd96 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,3 +62,6 @@ tag atomically. The version lives in three files — always change it with `python scripts/bump_version.py set --version X.Y.Z`, never by hand. See `docs/architecture/deployment.md` for the full pipeline + +## Gotchas & Rules +- **Check the library before writing a helper.** Before adding any function that renders, serializes, parses or walks a `DoclingDocument`, or splits chunks or fuses scores, check `docling_core.transforms.serializer.*` / `DocItem.export_to_*`, chonkie, and the DuckDB `lance` extension first. Phase 1 of retrieval shipped a `` serializer, a caption geometry join and a label→markdown CASE that docling already did; all were deleted. Functions with one consumer get inlined. From 2406ed8c98ed71bd1b53a873131430ad11ed8b14 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Mon, 7 Sep 2026 00:05:07 -0700 Subject: [PATCH 13/16] refactor(ocr): unwind the element layer; docling's chunker owns it Delete contexts/ocr/elements.py and its test, and revert arrow.py and layout_store.py (plus tests) to main: the items.markdown column, the MarkdownDocSerializer at parse time, and the Lance schema-widening path (_SQL_TYPES / _add_missing_columns) existed only to feed the element projection. docling_core's HybridChunker derives everything that layer computed from the stored DoclingDocument itself: heading breadcrumbs (meta.headings), provenance back to items (meta.doc_items), caption folding, furniture skipping and table row windows. link_captions in docling_builder.py stays; it is the parse-time fix the chunker relies on. Deps: drop chonkie (no remaining consumer). Add semchunk and tree-sitter{,-python,-c,-javascript,-typescript} explicitly, since docling_core.transforms.chunker imports them but only the [chunking] extra declares them, and that extra also pulls in transformers, which we do not want in the image. Smoke script now imports HybridChunker instead. --- CLAUDE.md | 2 +- .../server/scripts/smoke_retrieval_deps.py | 13 +- extralit-server/pyproject.toml | 8 +- .../src/extralit_server/contexts/ocr/arrow.py | 14 - .../extralit_server/contexts/ocr/elements.py | 139 -------- .../contexts/ocr/layout_store.py | 21 -- .../tests/unit/contexts/ocr/test_arrow.py | 29 -- .../tests/unit/contexts/ocr/test_elements.py | 330 ------------------ .../unit/contexts/ocr/test_layout_store.py | 13 - extralit-server/uv.lock | 224 +++++++----- 10 files changed, 149 insertions(+), 644 deletions(-) delete mode 100644 extralit-server/src/extralit_server/contexts/ocr/elements.py delete mode 100644 extralit-server/tests/unit/contexts/ocr/test_elements.py diff --git a/CLAUDE.md b/CLAUDE.md index 4ffdedd96..25a696993 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -64,4 +64,4 @@ tag atomically. The version lives in three files — always change it with See `docs/architecture/deployment.md` for the full pipeline ## Gotchas & Rules -- **Check the library before writing a helper.** Before adding any function that renders, serializes, parses or walks a `DoclingDocument`, or splits chunks or fuses scores, check `docling_core.transforms.serializer.*` / `DocItem.export_to_*`, chonkie, and the DuckDB `lance` extension first. Phase 1 of retrieval shipped a `` serializer, a caption geometry join and a label→markdown CASE that docling already did; all were deleted. Functions with one consumer get inlined. +- **Check the library before writing a helper.** Before adding any function that renders, serializes, parses or walks a `DoclingDocument`, or splits chunks or fuses scores, check `docling_core.transforms.serializer.*` / `DocItem.export_to_*`, `docling_core.transforms.chunker`, and the DuckDB `lance` extension first. Phase 1 of retrieval shipped a `` serializer, a caption geometry join and a label→markdown CASE that docling already did; all were deleted. Functions with one consumer get inlined. diff --git a/extralit-server/docker/server/scripts/smoke_retrieval_deps.py b/extralit-server/docker/server/scripts/smoke_retrieval_deps.py index 21a1d0c4c..5bc327213 100644 --- a/extralit-server/docker/server/scripts/smoke_retrieval_deps.py +++ b/extralit-server/docker/server/scripts/smoke_retrieval_deps.py @@ -46,20 +46,17 @@ def check_tessdata() -> str: return f"tessdata at {prefix}" -def check_chonkie() -> str: - from chonkie import RecursiveChunker, RecursiveRules +def check_docling_chunker() -> str: + from importlib.metadata import version - chunker = RecursiveChunker(tokenizer="character", chunk_size=64, rules=RecursiveRules()) - if not chunker("one two three. four five six."): - raise RuntimeError("RecursiveChunker returned no chunks") - import chonkie + from docling_core.transforms.chunker.hybrid_chunker import HybridChunker # noqa: F401 - return f"chonkie {chonkie.__version__}" + return f"docling-core {version('docling-core')}" def main() -> int: failures = 0 - for check in (check_lance_extension, check_liteparse, check_tessdata, check_chonkie): + for check in (check_lance_extension, check_liteparse, check_tessdata, check_docling_chunker): name = check.__name__.removeprefix("check_") try: print(f"ok {name}: {check()}") diff --git a/extralit-server/pyproject.toml b/extralit-server/pyproject.toml index 29a1ea3c8..2af2485e3 100644 --- a/extralit-server/pyproject.toml +++ b/extralit-server/pyproject.toml @@ -82,7 +82,13 @@ dependencies = [ "pdf-inspector>=1.14.2", # abi3 wheels from 2.14, so the block extractor is not tied to one CPython "liteparse>=2.14.0", - "chonkie>=1.7.0", + # what docling_core.transforms.chunker imports; the [chunking] extra would add transformers + "semchunk>=2.2", + "tree-sitter>=0.25,<0.27", + "tree-sitter-python", + "tree-sitter-c", + "tree-sitter-javascript", + "tree-sitter-typescript", "xxhash>=3.6.0", "obstore>=0.11.0", ] diff --git a/extralit-server/src/extralit_server/contexts/ocr/arrow.py b/extralit-server/src/extralit_server/contexts/ocr/arrow.py index 38fd5eba6..60136d74a 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/arrow.py +++ b/extralit-server/src/extralit_server/contexts/ocr/arrow.py @@ -10,7 +10,6 @@ from typing import Any, Optional import pyarrow as pa -from docling_core.transforms.serializer.markdown import MarkdownDocSerializer, MarkdownParams from docling_core.types.doc import DoclingDocument from docling_core.types.doc.document import DocItem, TableItem @@ -30,7 +29,6 @@ ("charspan_start", pa.int32()), ("charspan_end", pa.int32()), ("text", pa.string()), - ("markdown", pa.string()), ("html", pa.string()), ] ) @@ -60,20 +58,9 @@ def _table_html(doc: DoclingDocument, item: DocItem) -> Optional[str]: return None -def _serialize(serializer: MarkdownDocSerializer, item: DocItem) -> Optional[str]: - try: - return serializer.serialize(item=item).text - except Exception: # NULL, not '': a row the renderer could not handle is not a blank one - return None - - def item_rows(doc: DoclingDocument, document_id: str) -> list[dict[str, Any]]: """Flatten every item's provenance into rows matching `ITEM_SCHEMA`.""" rows: list[dict[str, Any]] = [] - # Raw characters and no image placeholder: this column feeds a search index, not a reader. - markdown = MarkdownDocSerializer( - doc=doc, params=MarkdownParams(escape_html=False, escape_underscores=False, image_placeholder="") - ) for reading_order, (item, _level) in enumerate(doc.iterate_items(with_groups=False)): base = { @@ -85,7 +72,6 @@ def item_rows(doc: DoclingDocument, document_id: str) -> list[dict[str, Any]]: "level": getattr(item, "level", None), "reading_order": reading_order, "text": getattr(item, "text", None) or None, - "markdown": _serialize(markdown, item), "html": _table_html(doc, item), } diff --git a/extralit-server/src/extralit_server/contexts/ocr/elements.py b/extralit-server/src/extralit_server/contexts/ocr/elements.py deleted file mode 100644 index b62c5b79d..000000000 --- a/extralit-server/src/extralit_server/contexts/ocr/elements.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Typed view of the layout store's `items` rows, one step below chunking. - -`arrow.item_rows` flattens a `DoclingDocument` into columns; this reads those columns back as -the three units a chunker dispatches on — a markdown run, a table, or a figure with its caption -— so `contexts/retrieval` never has to know a docling label. Rows in, rows out: chunking re-runs -from the Lance dataset without re-parsing the PDF. - -Rendering is docling's, done once at projection time into the `markdown` column. What this adds -is the heading breadcrumb, as window functions over the `items` columns, so a whole workspace is -projected in the same columnar pass as a single document. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, Optional - -import pyarrow as pa -from docling_core.types.doc import DocItemLabel - -from extralit_server.contexts.ocr.docling_builder import PICTURE_LABELS, TABLE_LABELS - -if TYPE_CHECKING: - import duckdb - -MARKDOWN = "markdown" -TABLE = "table" -FIGURE = "figure" - -#: Running page furniture: repeated on every page, retrievable on none of them. -SKIPPED_LABELS = frozenset({DocItemLabel.PAGE_HEADER, DocItemLabel.PAGE_FOOTER}) - -#: Headings open a breadcrumb slot. A title sits above every section header, whatever its level. -TITLE_SLOT = 0 - -#: The deepest breadcrumb slot; anything deeper is nested under the same ancestor anyway. -MAX_HEADING_LEVEL = 6 - -ELEMENT_SCHEMA = pa.schema( - [ - ("document_id", pa.string()), - ("type", pa.string()), - ("content", pa.string()), - ("page_no", pa.int32()), - ("bbox", pa.list_(pa.float32(), 4)), - ("label", pa.string()), - ("level", pa.int8()), - ("item_ref", pa.string()), - ("reading_order", pa.int32()), - ("headings", pa.list_(pa.string())), - ("heading_level", pa.int8()), - ] -) - - -def _labels(labels: frozenset[DocItemLabel]) -> str: - """A label set as a SQL `IN` list. Enum values only — nothing here is caller input.""" - return "(" + ", ".join(f"'{label.value}'" for label in sorted(labels)) + ")" - - -def elements_sql(source: str = "items") -> str: - """The projection, as one statement over anything DuckDB can scan with the `items` columns.""" - slots = range(TITLE_SLOT, MAX_HEADING_LEVEL + 1) - # Per slot: the latest heading opened there, and the depth of the latest heading at or above - # it. The slot is still in scope only while those coincide. - breadcrumb = ",\n ".join( - f"last_value(CASE WHEN slot = {s} THEN own_text END IGNORE NULLS) OVER running AS heading_{s},\n " - f"last_value(CASE WHEN slot <= {s} THEN slot END IGNORE NULLS) OVER running AS newest_{s}" - for s in slots - ) - headings = ", ".join(f"CASE WHEN newest_{s} = {s} THEN heading_{s} END" for s in slots) - # Python's `str.strip`; DuckDB's bare `trim` takes spaces off and leaves newlines behind. - strip = r"regexp_replace({}, '^\s+|\s+$', '', 'g')" - - return f""" -WITH sliced AS ( - SELECT * EXCLUDE (label), - coalesce(label, '{DocItemLabel.TEXT.value}') AS label, - row_number() OVER (PARTITION BY document_id ORDER BY reading_order, prov_index) AS ord, - -- A charspan short of the whole text is one page's share of an item that spans a break. - coalesce(charspan_start > 0 OR charspan_end < length(text), false) AS partial, - {strip.format("CASE WHEN partial THEN text[charspan_start + 1 : charspan_end] ELSE coalesce(text, '') END")} - AS own_text, - -- Markdown is rendered per item, so a partial row falls back to its share of the raw text. - CASE - WHEN markdown = '' THEN '' - WHEN partial OR markdown IS NULL THEN own_text - ELSE {strip.format("markdown")} - END AS content, - CASE - WHEN own_text = '' THEN NULL - WHEN label = '{DocItemLabel.TITLE.value}' THEN {TITLE_SLOT} - WHEN label = '{DocItemLabel.SECTION_HEADER.value}' - THEN least(greatest(coalesce(level, 1), 1), {MAX_HEADING_LEVEL}) - END AS slot - FROM {source} -), -breadcrumbs AS ( - SELECT *, - {breadcrumb} - FROM sliced - WINDOW running AS ( - PARTITION BY document_id ORDER BY ord ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW - ) -) -SELECT - document_id, - CASE - WHEN label IN {_labels(TABLE_LABELS)} THEN '{TABLE}' - WHEN label IN {_labels(PICTURE_LABELS)} THEN '{FIGURE}' - ELSE '{MARKDOWN}' - END AS type, - content, page_no, bbox, label, level, self_ref AS item_ref, reading_order, - list_filter([{headings}], heading -> heading IS NOT NULL) AS headings, - newest_{MAX_HEADING_LEVEL} AS heading_level -FROM breadcrumbs -WHERE label NOT IN {_labels(SKIPPED_LABELS)} AND content <> '' -ORDER BY document_id, ord -""" - - -def elements_table(items: Any, *, connection: Optional[duckdb.DuckDBPyConnection] = None) -> pa.Table: - """Project `items` rows into elements, in reading order, one per provenance row. - - An item spanning a page break stays two elements with two bboxes rather than one element - claiming to be in two places. `items` is anything DuckDB registers — an Arrow table, a Lance - dataset — and every document in it is projected in the same pass. - """ - import duckdb - - own = connection is None - connection = connection or duckdb.connect() - try: - connection.register("_items", items) - table = connection.execute(elements_sql("_items")).to_arrow_table() - finally: - connection.unregister("_items") - if own: - connection.close() - return table.cast(ELEMENT_SCHEMA) diff --git a/extralit-server/src/extralit_server/contexts/ocr/layout_store.py b/extralit-server/src/extralit_server/contexts/ocr/layout_store.py index 6396035b9..c2b2f96e2 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/layout_store.py +++ b/extralit-server/src/extralit_server/contexts/ocr/layout_store.py @@ -44,18 +44,6 @@ _SCHEMAS = {ITEMS_DATASET: ITEM_SCHEMA, PAGES_DATASET: PAGE_SCHEMA} -# Lance's SQL type names for `add_columns`; anything else needs a real migration, not a NULL fill. -_SQL_TYPES = { - pa.string(): "string", - pa.bool_(): "boolean", - pa.int8(): "tinyint", - pa.int16(): "smallint", - pa.int32(): "int", - pa.int64(): "bigint", - pa.float32(): "float", - pa.float64(): "double", -} - def layout_root(workspace_name: str) -> tuple[str, Optional[dict[str, str]]]: """Root of the workspace's datasets, resolved exactly like every other artifact of it.""" @@ -150,14 +138,6 @@ def open(self, name: str) -> Optional[lance.LanceDataset]: def _write(self, name: str, data: pa.Table, mode: str) -> int: return lance.write_dataset(data, self.uri(name), mode=mode, storage_options=self.storage_options).version - def _add_missing_columns(self, dataset: lance.LanceDataset, schema: pa.Schema) -> lance.LanceDataset: - """Widen a dataset written under an older schema; new columns are NULL for the rows already there.""" - missing = [field for field in schema if field.name not in dataset.schema.names] - if not missing: - return dataset - dataset.add_columns({field.name: f"CAST(NULL AS {_SQL_TYPES[field.type]})" for field in missing}) - return lance.dataset(dataset.uri, storage_options=self.storage_options) - def _replace_one(self, name: str, document_id: UUID | str, data: pa.Table) -> int: dataset = self.open(name) if dataset is None: @@ -167,7 +147,6 @@ def _replace_one(self, name: str, document_id: UUID | str, data: pa.Table) -> in # Another worker created it between the open and the write; join it instead. dataset = lance.dataset(self.uri(name), storage_options=self.storage_options) - dataset = self._add_missing_columns(dataset, data.schema) dataset.delete(_document_filter(document_id)) if data.num_rows == 0: return lance.dataset(self.uri(name), storage_options=self.storage_options).version diff --git a/extralit-server/tests/unit/contexts/ocr/test_arrow.py b/extralit-server/tests/unit/contexts/ocr/test_arrow.py index 286e7d46c..71e8c431e 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_arrow.py +++ b/extralit-server/tests/unit/contexts/ocr/test_arrow.py @@ -145,35 +145,6 @@ def test_text_items_have_no_html(self, doc): assert rows["#/texts/2"]["html"] is None - def test_markdown_is_docling_rendering_of_each_item(self, doc): - rows = {r["self_ref"]: r for r in items_table(doc, DOCUMENT_ID).to_pylist()} - - assert rows["#/texts/0"]["markdown"] == "# A Paper" - assert rows["#/texts/1"]["markdown"] == "### Methods" - assert rows["#/texts/2"]["markdown"] == "Body text here." - assert rows["#/tables/0"]["markdown"].startswith("| Group") - assert rows["#/pictures/0"]["markdown"] == "" - - def test_markdown_keeps_the_characters_a_search_index_needs(self): - document = new_document("sample") - ctx = PageContext(page_no=1, size=Size(width=612, height=792)) - append_blocks( - document, ctx, [LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text="p_value < 0.05 & n_obs")] - ) - - rows = items_table(document, DOCUMENT_ID).to_pylist() - - assert rows[0]["markdown"] == "p_value < 0.05 & n_obs" - - def test_markdown_is_rendered_once_per_item_not_once_per_provenance(self, doc): - for item in doc.texts: - if item.self_ref == "#/texts/2": - item.prov.append(ProvenanceItem(page_no=2, bbox=bbox(t=10, b=30), charspan=(0, 0))) - - rows = [r for r in items_table(doc, DOCUMENT_ID).to_pylist() if r["self_ref"] == "#/texts/2"] - - assert [r["markdown"] for r in rows] == ["Body text here."] * 2 - def test_parent_ref_is_recorded(self, doc): rows = {r["self_ref"]: r for r in items_table(doc, DOCUMENT_ID).to_pylist()} diff --git a/extralit-server/tests/unit/contexts/ocr/test_elements.py b/extralit-server/tests/unit/contexts/ocr/test_elements.py deleted file mode 100644 index 90734459c..000000000 --- a/extralit-server/tests/unit/contexts/ocr/test_elements.py +++ /dev/null @@ -1,330 +0,0 @@ -"""Tests for the columnar element view of `items` rows.""" - -import pyarrow as pa -import pytest -from docling_core.types.doc import BoundingBox, CoordOrigin, DocItemLabel, Size - -from extralit_server.contexts.ocr.arrow import ITEM_SCHEMA, item_rows -from extralit_server.contexts.ocr.docling_builder import ( - LayoutBlock, - PageContext, - append_blocks, - new_document, -) -from extralit_server.contexts.ocr.elements import ( - ELEMENT_SCHEMA, - FIGURE, - MARKDOWN, - TABLE, - elements_table, -) -from extralit_server.contexts.ocr.tables import make_cell - -DOCUMENT_ID = "11111111-2222-3333-4444-555555555555" - - -def bbox(t: float, b: float, left: float = 10.0, right: float = 100.0) -> BoundingBox: - return BoundingBox(l=left, t=t, r=right, b=b, coord_origin=CoordOrigin.TOPLEFT) - - -def row(label, reading_order, text=None, markdown=None, **overrides): - """An `items` row. `markdown` defaults to `text`, the way docling renders plain prose.""" - base = { - "document_id": DOCUMENT_ID, - "self_ref": f"#/texts/{reading_order}", - "parent_ref": None, - "label": label, - "content_layer": "body", - "level": None, - "reading_order": reading_order, - "prov_index": 0, - "page_no": 1, - "bbox": [0.0, 0.0, 10.0, 10.0], - "coord_origin": "TOPLEFT", - "text": text, - "markdown": text if markdown is None else markdown, - "html": None, - "charspan_start": None, - "charspan_end": None, - } - base.update(overrides) - return base - - -def elements(rows) -> list[dict]: - """The projection as dicts — assertions read better than Arrow column slices.""" - return elements_table(pa.Table.from_pylist(list(rows), schema=ITEM_SCHEMA)).to_pylist() - - -class TestSchema: - def test_the_projection_conforms_to_the_declared_schema(self): - table = elements_table(pa.Table.from_pylist([row(DocItemLabel.TEXT, 0, text="x")], schema=ITEM_SCHEMA)) - - assert table.schema == ELEMENT_SCHEMA - - def test_an_empty_input_yields_an_empty_table_not_an_error(self): - table = elements_table(pa.Table.from_pylist([], schema=ITEM_SCHEMA)) - - assert table.num_rows == 0 - assert table.schema == ELEMENT_SCHEMA - - -class TestHeadingBreadcrumb: - def test_breadcrumb_deepens_and_unwinds_with_heading_level(self): - rows = [ - row(DocItemLabel.TITLE, 0, text="A Paper"), - row(DocItemLabel.SECTION_HEADER, 1, level=1, text="Methods"), - row(DocItemLabel.SECTION_HEADER, 2, level=2, text="Sampling"), - row(DocItemLabel.TEXT, 3, text="Deep body."), - row(DocItemLabel.SECTION_HEADER, 4, level=1, text="Results"), - row(DocItemLabel.TEXT, 5, text="Shallow body."), - ] - - assert [e["headings"] for e in elements(rows)] == [ - ["A Paper"], - ["A Paper", "Methods"], - ["A Paper", "Methods", "Sampling"], - ["A Paper", "Methods", "Sampling"], - # Results closes Methods and Sampling but never the title. - ["A Paper", "Results"], - ["A Paper", "Results"], - ] - - def test_a_heading_carries_itself_so_a_chunk_knows_its_own_path(self): - rows = [row(DocItemLabel.SECTION_HEADER, 0, level=1, text="Methods", markdown="## Methods")] - - assert elements(rows)[0]["headings"] == ["Methods"] - - def test_the_breadcrumb_is_the_raw_heading_text_not_its_markdown(self): - rows = [ - row(DocItemLabel.SECTION_HEADER, 0, level=1, text="Methods", markdown="## Methods"), - row(DocItemLabel.TEXT, 1, text="Body."), - ] - - assert [e["headings"] for e in elements(rows)] == [["Methods"], ["Methods"]] - - def test_heading_level_tracks_the_deepest_slot_in_scope(self): - rows = [ - row(DocItemLabel.TITLE, 0, text="A Paper"), - row(DocItemLabel.SECTION_HEADER, 1, level=2, text="Sampling"), - row(DocItemLabel.TEXT, 2, text="Body."), - ] - - assert [e["heading_level"] for e in elements(rows)] == [0, 2, 2] - - def test_prose_before_any_heading_has_no_breadcrumb(self): - element = elements([row(DocItemLabel.TEXT, 0, text="Orphan.")])[0] - assert element["headings"] == [] - assert element["heading_level"] is None - - def test_a_blank_heading_opens_no_slot(self): - rows = [ - row(DocItemLabel.SECTION_HEADER, 0, level=1, text=" ", markdown=""), - row(DocItemLabel.TEXT, 1, text="Body."), - ] - - assert [e["headings"] for e in elements(rows)] == [[]] - - def test_levels_past_h6_share_the_deepest_slot(self): - rows = [ - row(DocItemLabel.SECTION_HEADER, 0, level=7, text="Seven"), - row(DocItemLabel.SECTION_HEADER, 1, level=99, text="Ninety-nine"), - ] - - assert [e["headings"] for e in elements(rows)] == [["Seven"], ["Ninety-nine"]] - - -class TestContent: - def test_content_is_the_rendered_markdown_not_the_raw_text(self): - rows = [row(DocItemLabel.SECTION_HEADER, 0, level=1, text="Methods", markdown="## Methods")] - - element = elements(rows)[0] - - assert element["type"] == MARKDOWN - assert element["content"] == "## Methods" - - def test_surrounding_whitespace_is_stripped_including_newlines(self): - assert elements([row(DocItemLabel.TEXT, 0, text=" \n\tProse.\n ")])[0]["content"] == "Prose." - - def test_a_row_rendered_blank_is_dropped(self): - # A linked caption renders as '' on its own: docling folds it into the table or figure. - assert elements([row(DocItemLabel.CAPTION, 0, text="Figure 1.", markdown="")]) == [] - - def test_a_row_never_rendered_falls_back_to_its_text(self): - # NULL is a row written before the column existed, or one the renderer could not handle. - assert elements([row(DocItemLabel.TEXT, 0, text="Old prose.", markdown="")]) == [] - rows = [{**row(DocItemLabel.TEXT, 0, text="Old prose."), "markdown": None}] - - assert [e["content"] for e in elements(rows)] == ["Old prose."] - - def test_tables_and_figures_carry_their_rendering_whatever_their_text(self): - rows = [ - row(DocItemLabel.TABLE, 0, markdown="Table 1.\n\n| N |\n|---|\n| 1 |"), - row(DocItemLabel.PICTURE, 1, markdown="Figure 1."), - row(DocItemLabel.PICTURE, 2, markdown=""), - ] - - assert [(e["type"], e["content"]) for e in elements(rows)] == [ - (TABLE, "Table 1.\n\n| N |\n|---|\n| 1 |"), - (FIGURE, "Figure 1."), - ] - - -class TestProvenance: - def test_running_headers_and_footers_are_dropped(self): - rows = [ - row(DocItemLabel.PAGE_HEADER, 0, text="Journal of Things"), - row(DocItemLabel.TEXT, 1, text="Real body."), - row(DocItemLabel.PAGE_FOOTER, 2, text="7"), - ] - - assert [e["content"] for e in elements(rows)] == ["Real body."] - - def test_one_element_per_provenance_row_when_an_item_spans_a_page_break(self): - text = "first half second half" - spanning = [ - row(DocItemLabel.TEXT, 0, page_no=1, text=text, charspan_start=0, charspan_end=11), - row(DocItemLabel.TEXT, 0, page_no=2, prov_index=1, text=text, charspan_start=11, charspan_end=22), - ] - - assert [(e["page_no"], e["content"]) for e in elements(spanning)] == [(1, "first half"), (2, "second half")] - - def test_a_page_spanning_heading_keeps_its_slice_and_loses_its_marker(self): - # Markdown is rendered per item and cannot be sliced by charspan, so the raw slice wins. - text = "Methods and Materials" - spanning = [ - row( - DocItemLabel.SECTION_HEADER, - 0, - level=1, - text=text, - markdown=f"## {text}", - charspan_start=0, - charspan_end=7, - ), - row( - DocItemLabel.SECTION_HEADER, - 0, - level=1, - page_no=2, - prov_index=1, - text=text, - markdown=f"## {text}", - charspan_start=7, - charspan_end=21, - ), - ] - - assert [e["content"] for e in elements(spanning)] == ["Methods", "and Materials"] - - def test_a_page_spanning_table_repeats_its_rendering_on_both_pages(self): - spanning = [ - row(DocItemLabel.TABLE, 0, page_no=1, markdown="| N |", charspan_start=0, charspan_end=0), - row(DocItemLabel.TABLE, 0, page_no=2, prov_index=1, markdown="| N |", charspan_start=0, charspan_end=0), - ] - - assert [(e["page_no"], e["content"]) for e in elements(spanning)] == [(1, "| N |"), (2, "| N |")] - - def test_elements_come_back_in_reading_order_whatever_the_row_order(self): - rows = [row(DocItemLabel.TEXT, 2, text="third"), row(DocItemLabel.TEXT, 0, text="first")] - - assert [e["content"] for e in elements(rows)] == ["first", "third"] - - def test_bbox_and_item_ref_survive_the_round_trip(self): - element = elements([row(DocItemLabel.TEXT, 0, text="x", bbox=[1.0, 2.0, 3.0, 4.0])])[0] - - assert element["bbox"] == [1.0, 2.0, 3.0, 4.0] - assert element["item_ref"] == "#/texts/0" - - -class TestManyDocuments: - def test_documents_are_projected_in_one_pass_without_bleeding_into_each_other(self): - rows = [] - for document_id in ("aaaa", "bbbb"): - for base in ( - row(DocItemLabel.SECTION_HEADER, 0, level=1, text=f"{document_id} heading"), - row(DocItemLabel.TEXT, 1, text=f"{document_id} body"), - ): - rows.append({**base, "document_id": document_id}) - - found = elements(reversed(rows)) - - assert [(e["document_id"], e["headings"]) for e in found] == [ - ("aaaa", ["aaaa heading"]), - ("aaaa", ["aaaa heading"]), - ("bbbb", ["bbbb heading"]), - ("bbbb", ["bbbb heading"]), - ] - - -class TestAgainstRealProjection: - @pytest.fixture - def found(self): - document = new_document("sample") - append_blocks( - document, - PageContext(page_no=1, size=Size(width=612, height=792)), - [ - LayoutBlock(label=DocItemLabel.TITLE, bbox=bbox(t=1, b=5), text="A Paper"), - LayoutBlock(label=DocItemLabel.SECTION_HEADER, bbox=bbox(t=10, b=40), text="Methods", level=1), - LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=50, b=70), text="Body text."), - LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=190, b=199), text="Table 1. Counts."), - LayoutBlock( - label=DocItemLabel.TABLE, - bbox=bbox(t=200, b=400), - cells=[make_cell("N", row=0, col=0, column_header=True), make_cell("42", row=1, col=0)], - ), - LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=500, b=600)), - LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=601, b=610), text="Figure 1. A red square."), - LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=700, b=750)), - ], - ) - return elements(item_rows(document, DOCUMENT_ID)) - - def test_captions_ride_inside_their_table_and_figure_not_the_prose(self, found): - assert [e["type"] for e in found] == [MARKDOWN, MARKDOWN, MARKDOWN, TABLE, FIGURE] - assert found[3]["content"].startswith("Table 1. Counts.\n\n|") - assert "|-----|" in found[3]["content"] and "42" in found[3]["content"] - assert found[4]["content"] == "Figure 1. A red square." - - def test_an_uncaptioned_figure_yields_nothing_retrievable(self, found): - assert sum(e["type"] == FIGURE for e in found) == 1 - - def test_docling_reserves_the_top_heading_for_the_title(self, found): - assert [e["content"] for e in found[:2]] == ["# A Paper", "## Methods"] - - def test_the_breadcrumb_reaches_every_element(self, found): - assert all(e["headings"] == ["A Paper", "Methods"] for e in found[1:]) - - -class TestTableChunking: - """chonkie owns the row-window split; pin the behaviour the element design leans on.""" - - def test_a_table_split_into_row_windows_repeats_caption_and_header_on_every_chunk(self): - from chonkie import TableChunker - - document = new_document("sample") - append_blocks( - document, - PageContext(page_no=1, size=Size(width=612, height=792)), - [ - LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=190, b=199), text="Table 1. Counts."), - LayoutBlock( - label=DocItemLabel.TABLE, - bbox=bbox(t=200, b=400), - cells=[ - make_cell("Group", row=0, col=0, column_header=True), - make_cell("N", row=0, col=1, column_header=True), - *[make_cell(f"g{i}", row=i, col=0) for i in range(1, 7)], - *[make_cell(str(i), row=i, col=1) for i in range(1, 7)], - ], - ), - ], - ) - content = elements(item_rows(document, DOCUMENT_ID))[0]["content"] - - chunks = [chunk.text for chunk in TableChunker(chunk_size=2)(content)] - - assert len(chunks) > 1 - assert all("Table 1. Counts." in chunk and "| Group" in chunk for chunk in chunks) - assert "| g6" in chunks[-1] and "| g6" not in chunks[0] diff --git a/extralit-server/tests/unit/contexts/ocr/test_layout_store.py b/extralit-server/tests/unit/contexts/ocr/test_layout_store.py index 846c99615..840870c86 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_layout_store.py +++ b/extralit-server/tests/unit/contexts/ocr/test_layout_store.py @@ -6,7 +6,6 @@ import time from uuid import uuid4 -import lance import pyarrow as pa import pytest @@ -46,7 +45,6 @@ def items(document_id: str, count: int = 3, label: str = "text") -> pa.Table: "charspan_start": 0, "charspan_end": 5, "text": f"row {i}", - "markdown": f"row {i}", "html": None, } for i in range(count) @@ -131,17 +129,6 @@ def test_a_zero_row_document_still_clears_its_old_rows(self, local_store): assert local_store.load_items(document_id).num_rows == 0 assert local_store.load_pages(document_id).num_rows == 0 - def test_a_dataset_written_under_an_older_schema_is_widened_not_rejected(self, local_store): - old, new = str(uuid4()), str(uuid4()) - narrow = items(old, 2).drop_columns(["markdown"]) - lance.write_dataset(narrow, local_store.items_uri(), mode="create") - - local_store.replace_document(new, items(new, 1), pages(new)) - - assert set(local_store.open(ITEMS_DATASET).schema.names) == set(ITEM_SCHEMA.names) - assert local_store.load_items(old, columns=["markdown"]).to_pylist() == [{"markdown": None}] * 2 - assert local_store.load_items(new, columns=["markdown"]).to_pylist() == [{"markdown": "row 0"}] - def test_replace_reports_the_dataset_versions(self, local_store): document_id = str(uuid4()) diff --git a/extralit-server/uv.lock b/extralit-server/uv.lock index 16c719caa..d3e9900ee 100644 --- a/extralit-server/uv.lock +++ b/extralit-server/uv.lock @@ -608,58 +608,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/68/687187c7e26cb24ccbd88e5069f5ef00eba804d36dde11d99aad0838ab45/charset_normalizer-3.4.6-py3-none-any.whl", hash = "sha256:947cf925bc916d90adba35a64c82aace04fa39b46b52d4630ece166655905a69", size = 61455, upload-time = "2026-03-15T18:53:23.833Z" }, ] -[[package]] -name = "chonkie" -version = "1.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "chonkie-core" }, - { name = "httpx" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "tenacity" }, - { name = "tokie" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c4/67/e6ceace2b2066cc38206c81e9105735975d7a58d89c9e7a871fe9acef63d/chonkie-1.7.0.tar.gz", hash = "sha256:4352aa214b66376c32523e524b94b7f1456a5e7611c48037dadb45e46f436b18", size = 190834, upload-time = "2026-07-07T10:06:02.475Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/c0/b5f91fb340d354a8c20e2f65d08b9dab357360da78d230e96c6d79b36a2e/chonkie-1.7.0-py3-none-any.whl", hash = "sha256:98822ca80fbf3c0775f59329a4f6b6bfedb34edff004b5e04e53c3281369b921", size = 233764, upload-time = "2026-07-07T10:06:00.824Z" }, -] - -[[package]] -name = "chonkie-core" -version = "0.10.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/17/ad31bbcfe1a7b63f76e065684a8e9cd98552d3c627fb28d8fbc05f1a9456/chonkie_core-0.10.2.tar.gz", hash = "sha256:c8e40ef8f3a034a7c5dd23a0401dce2ef2b4883f5a6a29cf94176d64b209bdbb", size = 69966, upload-time = "2026-05-28T19:16:12.4Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/a0/16ff6f6d3ddfb3f39d89663717e91eaa1bd5ba6ac52ec2fa01795ac33c65/chonkie_core-0.10.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b63d25a3270a98eb5a6739072315f36566c87139078797a0970bc8d480f5bbe0", size = 349233, upload-time = "2026-05-28T19:15:38.805Z" }, - { url = "https://files.pythonhosted.org/packages/ed/9d/2ed7af6666a168fcb18328e06354035f506aa8965a479827aa6f5528b189/chonkie_core-0.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:86c3e04603ce69d380f3f6a8f14f85dd902410ad1ec86709ddcfad43dc968946", size = 339167, upload-time = "2026-05-28T19:15:40.388Z" }, - { url = "https://files.pythonhosted.org/packages/12/bd/373ca97e84a4c4caf9adba9e710bc48f50c87524379a949cd17ecc588def/chonkie_core-0.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2225d6e6622b26d4542ed9661e71b51dfd8b4c5ae7e652fe85b4f0faa4be8eba", size = 390116, upload-time = "2026-05-28T19:15:41.556Z" }, - { url = "https://files.pythonhosted.org/packages/09/62/c3ae371065d31e52eb96d05ef02384521c99dd6bd036bc4b1d7a09ea4cbb/chonkie_core-0.10.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:e7e05701463e8e7a339a7be617fe2306153dcb5f269f43251e1516f40ce91134", size = 375235, upload-time = "2026-05-28T19:15:42.763Z" }, - { url = "https://files.pythonhosted.org/packages/16/08/3f55dbb5cd033b9c33e9e5fbaa5ee88af3dbc78e6858cf42341c903c5cc6/chonkie_core-0.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:c7c2c431d86fbae7c1f8d0b8628236df27b4dab040dc6504ded209be9e36f5b0", size = 230868, upload-time = "2026-05-28T19:15:44.207Z" }, - { url = "https://files.pythonhosted.org/packages/c9/0b/4c28270b24d9e756c90a52966f26ddff5fa2f639aed7e6cd9cc1e1728176/chonkie_core-0.10.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a431af90ac4e4072e02699ac1300efcaaff2ee432b052fe51027998859af1055", size = 349504, upload-time = "2026-05-28T19:15:45.801Z" }, - { url = "https://files.pythonhosted.org/packages/93/a6/069b46520a23264585ddacd117bee62594afff0f851cd1dfedf32b900faf/chonkie_core-0.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:712ecc448302bdc8f3ab4aab1e17d5d37b50d57d89d4fa4ec44821a37eb8fa57", size = 339673, upload-time = "2026-05-28T19:15:46.992Z" }, - { url = "https://files.pythonhosted.org/packages/a5/05/513baf0c159fb93e6c05bf199cca79045b2c186e9fea1fd7ebf47d063c7b/chonkie_core-0.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff79cbe5016f85a8fac2164e1716f3335b631704bfa486f0d1de02c6d88c2bce", size = 390030, upload-time = "2026-05-28T19:15:48.434Z" }, - { url = "https://files.pythonhosted.org/packages/50/65/4a86e1648147df94bcc2d681b6c5f40a21c57df65b45bf2f3ceb72fcb784/chonkie_core-0.10.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:f69969fa3f75930bd10f1ec502993b758df4e5a62d170c373a4fa73743dc2f93", size = 374622, upload-time = "2026-05-28T19:15:49.836Z" }, - { url = "https://files.pythonhosted.org/packages/54/62/425bca737db62a371b1e3393d76f69f483baf3482ccd88504089501ee0e4/chonkie_core-0.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:cbe5a8a1e89a79a74bac99409e1bf5e7a5b3e2286ac5f2185077d39f5fa95175", size = 231028, upload-time = "2026-05-28T19:15:50.956Z" }, - { url = "https://files.pythonhosted.org/packages/56/2f/4dd88b4af9ef0e9ed31a3c6a3d8e44327c5ca22b074076830b0224e0ad58/chonkie_core-0.10.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:31f8fa3415d2e93cc3bdb1e99c3b6865278169af348c68fb2943aed2fcabc010", size = 347520, upload-time = "2026-05-28T19:15:52.331Z" }, - { url = "https://files.pythonhosted.org/packages/b2/3f/ba4083f21b4ef52ed130a79f371fed453b7edf998bf08079694137880bff/chonkie_core-0.10.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a6d081363fbfaad0f36c66e67e6dec1c9f9850c04c0475b64b3e47084257b646", size = 336897, upload-time = "2026-05-28T19:15:53.787Z" }, - { url = "https://files.pythonhosted.org/packages/0d/9f/cbfec0f25e35d2a5b62a9cba7a5a85cfc5349210b9f9acb40832060211e4/chonkie_core-0.10.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51dbf2163dac5ad695dde3f92155c6febb9332b5fb5a174ac80ae9fc8f8ccde9", size = 387231, upload-time = "2026-05-28T19:15:54.851Z" }, - { url = "https://files.pythonhosted.org/packages/89/c8/b0d91419730bff4566305f2c85f27fb1378e8e1e99f8e59403c6b25d43a2/chonkie_core-0.10.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5a09bcb56d9d9f2fa6ec84485aeecbcaa375567eea9fbec18517d2ef643b9029", size = 371506, upload-time = "2026-05-28T19:15:56.172Z" }, - { url = "https://files.pythonhosted.org/packages/8c/c2/05c2c85b11d092f04768c41fb9cfdd27a9d27fa7e10e107188a51ce8813f/chonkie_core-0.10.2-cp312-cp312-win_amd64.whl", hash = "sha256:ac686d192c4dd7e038cea4aba59c677d45e2b1e8de8eb1dd45d269b22a4ae201", size = 229686, upload-time = "2026-05-28T19:15:57.759Z" }, - { url = "https://files.pythonhosted.org/packages/59/4f/0814d2bd9abddbbc37f8c204f056d28f2ab7d14bf742515e9684062bd758/chonkie_core-0.10.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:25372879e43b235dee01f48309634a3fce07b55c382a7fbe200d6b66769267e4", size = 346848, upload-time = "2026-05-28T19:15:58.979Z" }, - { url = "https://files.pythonhosted.org/packages/3f/33/b640f406b6d2fe30d5c41d4a91411d9ed1dc22acf6497cda837a6f121ba4/chonkie_core-0.10.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3da60e797b2fcbef3e17fad3e46bb7d4f5541ac5f87a5556c2c610cef83a549a", size = 336613, upload-time = "2026-05-28T19:16:00.125Z" }, - { url = "https://files.pythonhosted.org/packages/e9/76/079296e83d2ed6e34b8ed24bd23e59193b502b53ddfa176b3e921410a5db/chonkie_core-0.10.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aebeba2d2cb0d30eefdbdb18d2ab2fcfbc0ea0d5121f61f537a6114a1da56101", size = 386412, upload-time = "2026-05-28T19:16:01.425Z" }, - { url = "https://files.pythonhosted.org/packages/04/6e/2c505276878b695b4cee07eb35a5c536ed92d60c722b4547da1517fb3e41/chonkie_core-0.10.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:72974163eab058d14c365f45b53d2367b1e04a0dcd779dedd828973cf21256d7", size = 371726, upload-time = "2026-05-28T19:16:02.8Z" }, - { url = "https://files.pythonhosted.org/packages/38/39/158f85b728d84e85a00768ccebe3a289d21462f52ba7850db9e01e61e4df/chonkie_core-0.10.2-cp313-cp313-win_amd64.whl", hash = "sha256:9f77811bb722bd019a52353364bc6dcea1a9998120a413f1c2a1c79153a66386", size = 229447, upload-time = "2026-05-28T19:16:04.21Z" }, - { url = "https://files.pythonhosted.org/packages/0b/45/e4d68840847133afa9c69fc0e28575270afdd9dcc0c13280a9b40445377d/chonkie_core-0.10.2-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:cb0626fa4d9df5128188eef069f69809571fbfc6db47e348d5f4d1199fa34b9c", size = 346802, upload-time = "2026-05-28T19:16:05.415Z" }, - { url = "https://files.pythonhosted.org/packages/26/4d/29e42ac094624a0ebfe74684ce62eb870e640679c1feb8da63e631a45cde/chonkie_core-0.10.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5266bdb8887c223782096c97a72eb49f19930f797213fa65b82199042742dc98", size = 336470, upload-time = "2026-05-28T19:16:06.575Z" }, - { url = "https://files.pythonhosted.org/packages/56/05/b3e206f063e6515a1b2e1f873ff33d91bf1eaab99554577c47804ebbbf37/chonkie_core-0.10.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4b6e2cf0411cb675d8b22f2b234b07e20724ded6cf84b2bc625b677885d2ef7", size = 386170, upload-time = "2026-05-28T19:16:07.761Z" }, - { url = "https://files.pythonhosted.org/packages/78/1e/4c775aec3d7ef6ff6ad90570c5b6f612fda9bf1ae7dfb4bd322d0fe55ffe/chonkie_core-0.10.2-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:d1999d993df8ecf941361b8eff9618f2a9bf3436b4a1851ec3f3f42d9a9c1a7e", size = 370624, upload-time = "2026-05-28T19:16:08.883Z" }, - { url = "https://files.pythonhosted.org/packages/f5/d0/103da313f8990607ac77c4735f372734a4d96afe79fdabc5883731bb2063/chonkie_core-0.10.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:acf83c774d5646dd9f16b896722155db8e87948fc93ee952fd649b5344d9205d", size = 101801, upload-time = "2026-05-28T19:16:10.226Z" }, - { url = "https://files.pythonhosted.org/packages/b0/93/e3167b1823f919f1a4c517092f73dc4107737de8bb53c67ff60c964b5244/chonkie_core-0.10.2-cp314-cp314-win_amd64.whl", hash = "sha256:20c9d64b9c5169d1f7e5ceeaa22bff3613017b8937c224ba85e3e24748fb0ba3", size = 228655, upload-time = "2026-05-28T19:16:11.357Z" }, -] - [[package]] name = "click" version = "8.3.1" @@ -1079,7 +1027,6 @@ dependencies = [ { name = "authlib" }, { name = "bcrypt" }, { name = "brotli-asgi" }, - { name = "chonkie" }, { name = "click" }, { name = "datasets" }, { name = "docling-core" }, @@ -1117,10 +1064,16 @@ dependencies = [ { name = "pyyaml" }, { name = "rich" }, { name = "rq" }, + { name = "semchunk" }, { name = "social-auth-core" }, { name = "sqlalchemy" }, { name = "standardwebhooks" }, { name = "tenacity" }, + { name = "tree-sitter" }, + { name = "tree-sitter-c" }, + { name = "tree-sitter-javascript" }, + { name = "tree-sitter-python" }, + { name = "tree-sitter-typescript" }, { name = "typer" }, { name = "uvicorn", extra = ["standard"] }, { name = "xxhash" }, @@ -1158,7 +1111,6 @@ requires-dist = [ { name = "authlib", specifier = ">=1.6.9" }, { name = "bcrypt", specifier = ">=4.2.0" }, { name = "brotli-asgi", specifier = ">=1.4.0" }, - { name = "chonkie", specifier = ">=1.7.0" }, { name = "click", specifier = ">=8.2.0" }, { name = "datasets", specifier = ">=3.6.0" }, { name = "docling-core", specifier = ">=2.91.0,<3.0.0" }, @@ -1197,10 +1149,16 @@ requires-dist = [ { name = "pyyaml", specifier = ">=5.4.1,<6.1.0" }, { name = "rich", specifier = "!=13.1.0" }, { name = "rq", specifier = ">=2.4.1" }, + { name = "semchunk", specifier = ">=2.2" }, { name = "social-auth-core", specifier = ">=4.5.0" }, { name = "sqlalchemy", specifier = ">=2.0.0" }, { name = "standardwebhooks", specifier = ">=1.0.0" }, { name = "tenacity", specifier = ">=9.1.2" }, + { name = "tree-sitter", specifier = ">=0.25,<0.27" }, + { name = "tree-sitter-c" }, + { name = "tree-sitter-javascript" }, + { name = "tree-sitter-python" }, + { name = "tree-sitter-typescript" }, { name = "typer", specifier = ">=0.19.1" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.32.0" }, { name = "xxhash", specifier = ">=3.6.0" }, @@ -3972,6 +3930,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/e8/726643a3ea68c727da31570bde48c7a10f1aa60eddd628d94078fec586ff/ruff-0.15.7-py3-none-win_arm64.whl", hash = "sha256:18e8d73f1c3fdf27931497972250340f92e8c861722161a9caeb89a58ead6ed2", size = 11023304, upload-time = "2026-03-19T16:26:51.669Z" }, ] +[[package]] +name = "semchunk" +version = "4.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill", marker = "sys_platform == 'win32'" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b4/ab/74030fb91d965e3feecaad2bea1fbc5adeef8836cc5f15aab492d6bc0319/semchunk-4.1.1.tar.gz", hash = "sha256:f27dc85716a0e9509a3f99c78b961cc476b3120efe711ee9a9a950ed1bd0f8ba", size = 24910, upload-time = "2026-06-13T00:45:02.446Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/67/892571d1a035a146c8f3cad38306b90cb6a7cd4e042aa8e5554144757d9e/semchunk-4.1.1-py3-none-any.whl", hash = "sha256:acbc535e89824dce954a554e297438b2ee3506b71505f1472eb2bf8057fcba3c", size = 18706, upload-time = "2026-06-13T00:45:00.974Z" }, +] + [[package]] name = "shellingham" version = "1.5.4" @@ -4212,40 +4183,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/59/8c/b1c87148aa15e099243ec9f0cf9d0e970cc2234c3257d558c25a2c5304e6/tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f01a9c019878532f98927d2bacb79bbb404b43d3437455522a00a30718cdedb5", size = 3373542, upload-time = "2026-01-05T10:40:52.803Z" }, ] -[[package]] -name = "tokie" -version = "0.1.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/18/7e/ccb0f35fcdaea03ea4db7b4f6066aeec63d71bebf83d8181cda2fa320661/tokie-0.1.4.tar.gz", hash = "sha256:c8707df026d79a13228d410d607fde0dff5860d0f45ce11d89d85fb0a4c2a114", size = 280777, upload-time = "2026-07-24T16:41:15.502Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/f8/6507e5ef1fa3ccf8d8b453787ae5e8dd3214bdf925194442adad9b72e71c/tokie-0.1.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:80ff46843015605d2c3e34889f68a370d41d0f257e0edee4a4c253916b9eb3ba", size = 2866087, upload-time = "2026-07-24T16:40:35.314Z" }, - { url = "https://files.pythonhosted.org/packages/d0/3e/2b23f018a1bced6603a049c2c73b6bf902da17a5956863847b81b06c2133/tokie-0.1.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:299a87141ad6802fb48c7146dae3496c3716fc54f99e2a909b32c742780fa03a", size = 2897113, upload-time = "2026-07-24T16:40:37.475Z" }, - { url = "https://files.pythonhosted.org/packages/94/b0/c3bb8317d854c6da6a56f646bfd99ef7632c71b837462f06c8af15dbbebf/tokie-0.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:450e7270206afad356cad1e7d9c83cda98e5a0a8e633957df0abc484ddc08589", size = 3095044, upload-time = "2026-07-24T16:40:38.995Z" }, - { url = "https://files.pythonhosted.org/packages/3e/f5/839989186caa2769936f1e0a5dac7d33e6d73a46a958a164f8292cecc887/tokie-0.1.4-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:440c992d28f457f79ff244f1b8291fa0e245d058f131619dee728b99ba7e5eb8", size = 3207176, upload-time = "2026-07-24T16:40:40.588Z" }, - { url = "https://files.pythonhosted.org/packages/5b/0e/5c6cf9a817513dcc081c922a333b0a54c2d9ad506d32ba2d31a1f864e40d/tokie-0.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:0bc02cdd9bd9a15d997a7b5f0beb3e148d9c3efd5d949b9f51077f6c53aef6d1", size = 2639034, upload-time = "2026-07-24T16:40:42.367Z" }, - { url = "https://files.pythonhosted.org/packages/19/b6/53cd60a1182ce475812ae38ecaff15cfba4b447976920289f2caf471b14c/tokie-0.1.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4d922e3ba6635ecd513cfd6b8cb0969e88c0b329a41cbc0dd0c1a2c9053800f0", size = 2865741, upload-time = "2026-07-24T16:40:43.852Z" }, - { url = "https://files.pythonhosted.org/packages/ca/b8/cf7a51c9d08bd2e5f4f8510c4da20b558bb8056e215a6045333308c07f69/tokie-0.1.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d06278ae88011f0fc6ff99d36707c856f633131f94dfc35f2033d0a71107d4bf", size = 2897068, upload-time = "2026-07-24T16:40:45.457Z" }, - { url = "https://files.pythonhosted.org/packages/b0/04/e447decd69987e3f4c00015ed4cd08106d851fb9ab2e9f34643d5cff7df4/tokie-0.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8a5b5dd143d807981711e973a9fff21a4fc48e472ef3a976054be04ac236a63", size = 3094709, upload-time = "2026-07-24T16:40:46.855Z" }, - { url = "https://files.pythonhosted.org/packages/de/d9/3b10e0c2c753db3b85a55d891548796f2fc16e5000941167ff2c6f3c6428/tokie-0.1.4-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d3ef8dcf454b1601d80ef8cfb5c5b86f89c8dd26f68dd05a5c25248872d20039", size = 3206948, upload-time = "2026-07-24T16:40:48.266Z" }, - { url = "https://files.pythonhosted.org/packages/91/ed/41aeae9d70bb7484ca4e4a9166d5330781b4a59b3cb8f1413a6b62e28204/tokie-0.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:885691c40fe47b1daaff1e7c9d13cc2f9734c8cd089a00fee5d05ee0f9a703d0", size = 2638905, upload-time = "2026-07-24T16:40:49.904Z" }, - { url = "https://files.pythonhosted.org/packages/7b/af/fae60b61803429d29f0ab5d01e474ff8461496d91c6c7753a7cede5d8db7/tokie-0.1.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3105e5d0df6829fddc0a6761e726dcae80cb8b373f39fe6396a32a2014157478", size = 2865113, upload-time = "2026-07-24T16:40:52.305Z" }, - { url = "https://files.pythonhosted.org/packages/4a/3f/ee12c8b77393ce6ceb5c465908b43ab96af4499fff88bff7e16f13022b09/tokie-0.1.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4ad390232d70e8e4b239e6e7edd8578e56433014ad496ea3597faa677b5e18b3", size = 2889620, upload-time = "2026-07-24T16:40:53.665Z" }, - { url = "https://files.pythonhosted.org/packages/bf/f4/32eaa1aa37a5ad86244a1315096401cccefbb82c2c723b7181a42ee668e0/tokie-0.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eecd300fa19704fb3022aaedf8c1eafa9d9825c87ee13d2203e383bab4e764f", size = 3089894, upload-time = "2026-07-24T16:40:55.126Z" }, - { url = "https://files.pythonhosted.org/packages/62/66/b630a22d546da4712765e0472c4dbfdf50dcfc77b821ad274b12ee7be469/tokie-0.1.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:bbab9991a569beb667a0fe5f7c5698050914a46c015d0d323d8f4ef9e8c8aee2", size = 3202845, upload-time = "2026-07-24T16:40:56.571Z" }, - { url = "https://files.pythonhosted.org/packages/4c/64/e2bf4606adb89c942f72524b2a5a832cb6899676a623df04adaea1eee5b0/tokie-0.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:b0d245887bb806c42593304abce545fd1e7808c091946a9f3923d9211e99f4ff", size = 2635997, upload-time = "2026-07-24T16:40:58.283Z" }, - { url = "https://files.pythonhosted.org/packages/96/15/3fe8af5fe7ee607a6f6817e1119000b8bc828c9322dd68cccc724334ce38/tokie-0.1.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:e28c1c825479c8783ef2917bcc75964113d16ece254f9cd83234d19f3eaf7ff3", size = 2864983, upload-time = "2026-07-24T16:40:59.627Z" }, - { url = "https://files.pythonhosted.org/packages/ea/3f/35246c2ea85eb773342c64521da2481b0ff363cf1b39b1118eb6ab1c2ce3/tokie-0.1.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:199738e086f4a32892e651d9f98d10e606d12d581563041183e5c1b37a83745b", size = 2890250, upload-time = "2026-07-24T16:41:01.144Z" }, - { url = "https://files.pythonhosted.org/packages/f0/6c/114dfa287748d1bc32a0ff2ea56c24284a8afa4b9a1a391c65d2b20fed90/tokie-0.1.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2c41b8565e22aa0692e4cfcab595c0fe8b7ee742874e884ab9df928aa16d70e", size = 3089343, upload-time = "2026-07-24T16:41:02.629Z" }, - { url = "https://files.pythonhosted.org/packages/82/45/3e0fb358bd6fc3a9166cb7472f48055d7e894188b6f7553ebbad9e4e81f6/tokie-0.1.4-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:b39bf238b7b89bce19fa8ee353581ecffed46a7aac038502b9649789854b7e51", size = 3202698, upload-time = "2026-07-24T16:41:04.36Z" }, - { url = "https://files.pythonhosted.org/packages/54/99/d2444072b3ad7f2bcb7c40286e71d39f1cb968ebc52a90e6de31d5e5782d/tokie-0.1.4-cp313-cp313-win_amd64.whl", hash = "sha256:3ded2223b0486c8cf7922b9cf632ff10ee334bb7c736094f257515115826de87", size = 2635505, upload-time = "2026-07-24T16:41:05.533Z" }, - { url = "https://files.pythonhosted.org/packages/0f/32/4a0f5a30c5f31868fe795f6230cfad64545ed4900ee98f9f16f688f9dbb7/tokie-0.1.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:29013b3b7b6a6f3ed4b2e1fe35eeb4702236088c59bf137f8cb85954d69d5e84", size = 2866161, upload-time = "2026-07-24T16:41:06.973Z" }, - { url = "https://files.pythonhosted.org/packages/e2/db/150c266d98d5093fb65c9d150c8594d598a2b2626e6c4b88907552659d00/tokie-0.1.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b80085348b77d7e783513bfdc79dd4452c0602a28e04907124dcc17cbed8e5", size = 2890795, upload-time = "2026-07-24T16:41:08.397Z" }, - { url = "https://files.pythonhosted.org/packages/4c/42/078cc772347e8b6279fd9b643b5c7fb47eb49fe4eef88e3b8bbe165bbffc/tokie-0.1.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25344e4c02e47fc8cd2a146b1aa4ea5286b478bcbc5b109b1ec1da8d0ac17ff1", size = 3090930, upload-time = "2026-07-24T16:41:09.851Z" }, - { url = "https://files.pythonhosted.org/packages/6b/3f/8267161d70197f25ebe33e01907cac6a427228357b4ff9653900020f0e1a/tokie-0.1.4-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:cbc7b468fba5d5f506b85bb4319ffe825ff2c328aaea3c8ff57a36740af00e6d", size = 3203012, upload-time = "2026-07-24T16:41:11.375Z" }, - { url = "https://files.pythonhosted.org/packages/78/36/b97834b2152c24e6c5a5aaa39b6e4659105b3b3c472877e354d19baf3237/tokie-0.1.4-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:9b90d2ac37476497b7f5de25df4208b83d6a4a3cf43c790b7b91dcbc5419da8b", size = 958833, upload-time = "2026-07-24T16:41:12.715Z" }, - { url = "https://files.pythonhosted.org/packages/85/9d/d727a3a0d05bb78cc839e77a46b00bf8542769384ac6aa64c953314d5bfc/tokie-0.1.4-cp314-cp314-win_amd64.whl", hash = "sha256:2d4fb5a6b9cd4e137c3e6429129a5901909414552f35236c213668fbeab6cc77", size = 2637400, upload-time = "2026-07-24T16:41:14.288Z" }, -] - [[package]] name = "tomli" version = "2.4.0" @@ -4312,6 +4249,117 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] +[[package]] +name = "tree-sitter" +version = "0.26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f7/03/5600b84aff2e6c4fe80cfebb4063fe2f50299521befe5f6092ab8c082f4a/tree_sitter-0.26.0.tar.gz", hash = "sha256:b40c219edccc4564530c96f8f1556f6202b37cda964d1cbd7bd2b7e68b40a245", size = 191423, upload-time = "2026-06-30T12:14:27.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/2f/201c33ea65875d8e4ec73e4d1949718ec49780d84c0adf19793ef75d99a2/tree_sitter-0.26.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ff527388df14cb5009f9274faf78cc69a7393ae6acf3b04784b8acca249519c5", size = 148676, upload-time = "2026-06-30T12:13:42.718Z" }, + { url = "https://files.pythonhosted.org/packages/9e/db/05b9d45dd2b9827bf91b6819e749227ca6d686d58658292c0f149294b18e/tree_sitter-0.26.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7bcbadfa614326debef581957d5c780a9d7f66065c13deea61aa21d1dd36263f", size = 140757, upload-time = "2026-06-30T12:13:44.007Z" }, + { url = "https://files.pythonhosted.org/packages/b9/08/1e1da65c1585b8d70130b26d65b41a71737ab623c1fab1008479c2b95b50/tree_sitter-0.26.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f941cea06128c1f74f8937a8e2a90c7db49cf4be6647cd9e07d92a306d91517", size = 631526, upload-time = "2026-06-30T12:13:45.008Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b7/06353044a80ee58a71e884b4a9b2913705849d81025d87308abdfef8f883/tree_sitter-0.26.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e9e46b664887d8c1014f1fb33e09454bbdd9ec1fe29b7fd02dde7b46bc1bb81a", size = 658688, upload-time = "2026-06-30T12:13:46.491Z" }, + { url = "https://files.pythonhosted.org/packages/df/56/c4b22ccbc4f89ae507c0b76e29f363ad4f16eb38c43f7392b3eb9afec64e/tree_sitter-0.26.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:763627db05db34f12333081bd7422cc1c675893d373cc870b3e9249e200700e4", size = 644399, upload-time = "2026-06-30T12:13:47.719Z" }, + { url = "https://files.pythonhosted.org/packages/b5/b7/6b3f0192d5b9b49a199cb0dcd5e45dd1327a82c52c80a49edd790e3a2d9b/tree_sitter-0.26.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:17a1c5cfd3a05d5c7c86bf4282b6ef8092c91dc0a98390499669c3fedb7d1814", size = 655316, upload-time = "2026-06-30T12:13:49.03Z" }, + { url = "https://files.pythonhosted.org/packages/a5/6b/f7475c8f8d699671c2a80c3ed16f5cddd161280c6ed5b845117179c66075/tree_sitter-0.26.0-cp310-cp310-win_amd64.whl", hash = "sha256:f289be0225ba2ace8e87d6c9639b2bc9ff2b5271afb7c5d39282a4a00e248682", size = 129494, upload-time = "2026-06-30T12:13:50.242Z" }, + { url = "https://files.pythonhosted.org/packages/f6/20/0df8dd708638cba7ef875fff4ce80122af7f604f1f0b566de2164108bc01/tree_sitter-0.26.0-cp310-cp310-win_arm64.whl", hash = "sha256:526a165a2cb1d1f79e247d400f0e0acd8d49a817d6f312d543513af200b1f886", size = 116486, upload-time = "2026-06-30T12:13:51.21Z" }, + { url = "https://files.pythonhosted.org/packages/41/18/78aae7e4b5a36daaebb0276e4b07d084d45298758000787838e89329e11f/tree_sitter-0.26.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1d6fe0e8fb4df77b5ee816228e2c4475a63d8cc1d4d3a7ffd7097b2b87fc3e95", size = 148679, upload-time = "2026-06-30T12:13:52.27Z" }, + { url = "https://files.pythonhosted.org/packages/24/e4/b371b9553b0e47d130fc2073e56cab94fecc868be04666bf5bbd1fcd1cc9/tree_sitter-0.26.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:514a9bf8993e5210e7970736aaf6020d1759b670e195ef17b1c48f586aa30736", size = 140759, upload-time = "2026-06-30T12:13:53.221Z" }, + { url = "https://files.pythonhosted.org/packages/22/7d/266fb0f2c41e6fb00b0f40e7a3338cdf99651e6a6511ca72bc78fc697636/tree_sitter-0.26.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10f0d4eb94aa7242dcb7f554bcd24dd7ba1c114f00d58759ba08c7a46c8ec51a", size = 637206, upload-time = "2026-06-30T12:13:54.334Z" }, + { url = "https://files.pythonhosted.org/packages/40/9f/47cf22febb47132d5b3a507a27bb99ef89fe5c8ec420a13c6daa9b64f782/tree_sitter-0.26.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:335294ce0504fcefde5245dff596778ffaf820205b98ae0b549c72e48855f1d8", size = 664758, upload-time = "2026-06-30T12:13:55.42Z" }, + { url = "https://files.pythonhosted.org/packages/4c/4d/8d144ca3beb46a62a5102b6deac76bb0da55235c2c7840faf3b12f2e9d97/tree_sitter-0.26.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f9997ba61368c48ed54e715676afadf703947a1542464e39d047764fb3624b01", size = 647438, upload-time = "2026-06-30T12:13:56.523Z" }, + { url = "https://files.pythonhosted.org/packages/4d/ed/ed1d6e78520c4fb64ed52fec3f2947bf8c1fbad7bc24e282c56193c9ba42/tree_sitter-0.26.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c56581ad256c4195a21bfe449fed5d44a02fe83a4a7d6e70e6ec302c881191c7", size = 661944, upload-time = "2026-06-30T12:13:57.82Z" }, + { url = "https://files.pythonhosted.org/packages/10/83/45f5bd43db1b8248d2fd08ef6cbe43e2725c539e09a2cfb8bc2818646788/tree_sitter-0.26.0-cp311-cp311-win_amd64.whl", hash = "sha256:0f8793fd18ad7eec276ed4b51c097b4bf2002b357259b66b0d75db1f3f41c754", size = 129496, upload-time = "2026-06-30T12:13:59.216Z" }, + { url = "https://files.pythonhosted.org/packages/f1/8d/be68e6c04563eb54145424cc83fe0aa8b0ba6c90d8989cf8a032671b5f16/tree_sitter-0.26.0-cp311-cp311-win_arm64.whl", hash = "sha256:dea4b4e27d49e9ec5b785d4f994da000e6726882fcc6ad05ec98478500c71aef", size = 116484, upload-time = "2026-06-30T12:14:00.147Z" }, + { url = "https://files.pythonhosted.org/packages/87/ca/565702c44815393e3a973552ad546db4e5ca081ca8698640b4e93d809f51/tree_sitter-0.26.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6cb2bd20efb2544c19ac54486ab7cb8ec7b36f913bbe1ce95df84acb96743d9c", size = 148934, upload-time = "2026-06-30T12:14:01.188Z" }, + { url = "https://files.pythonhosted.org/packages/54/6f/8bb61957f16ec1b1d92410a006cdc84a952b6352a7313b2ad299f2d21484/tree_sitter-0.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:918d89529786873f0982a0f59c2a303cd065fbfd1b903d71a8e4e1584f67b42e", size = 140820, upload-time = "2026-06-30T12:14:02.087Z" }, + { url = "https://files.pythonhosted.org/packages/78/0a/8a6f08559182643a814a4ab559948ae817b2851890fd9b995a4fff6541ce/tree_sitter-0.26.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30a88be89ff1f2755297f81e8080d88b795dd98720c3f9fa2acf93873182cc95", size = 638844, upload-time = "2026-06-30T12:14:03.428Z" }, + { url = "https://files.pythonhosted.org/packages/8a/2f/6e6781b31677231366cb3cf27bc8269157f6d4b03c9032865a4f5f2bbe7e/tree_sitter-0.26.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a6b333b0282d8bb0af741f9b018bd2523d4eecb2686bf6717066a625fecfaa4", size = 667487, upload-time = "2026-06-30T12:14:04.669Z" }, + { url = "https://files.pythonhosted.org/packages/02/0b/0483078c8567445557a7015b0e5b187f6d7d4fda73464df9c4bdea7f7f3c/tree_sitter-0.26.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f3c44339dd34fe8eb2b8d5aa7610660499a795f70376b130bbee7a437337280", size = 647975, upload-time = "2026-06-30T12:14:05.797Z" }, + { url = "https://files.pythonhosted.org/packages/27/68/da83ca72c984e96ab4eb3bee0db1a6ffb5de1c8c455f92bd9f420cde7f0e/tree_sitter-0.26.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:94550e13b6ae576969da40246f4c4abb206380b5375ad43f26dd9151d55438e3", size = 665018, upload-time = "2026-06-30T12:14:07.278Z" }, + { url = "https://files.pythonhosted.org/packages/d1/36/4d67927fd47b89af4a00f65f55a7370e28778cd50e972c2430487e3ecc27/tree_sitter-0.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:ca89e361a276dbc934b28a43dd881199e25d34ff5493ee0ce45f3c52a6124a37", size = 129619, upload-time = "2026-06-30T12:14:08.373Z" }, + { url = "https://files.pythonhosted.org/packages/ed/72/cdefad523eb78710679c6da6a79e3d90f5afd32b1c6aa5a17bac7eef99f6/tree_sitter-0.26.0-cp312-cp312-win_arm64.whl", hash = "sha256:bc6cb01d5ee75c85424aa1f1c72a82d8f07fd52539a0f3c4a6ed3e8721079b84", size = 116545, upload-time = "2026-06-30T12:14:09.273Z" }, + { url = "https://files.pythonhosted.org/packages/cb/b0/465257cf8f972ad9f9812ec1cbaa8ec210ebebb601ade9a15881aa2436b4/tree_sitter-0.26.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ed0889dbed843ce45ede9f5169c0b2dea2222f12685844a03fadb81f12705867", size = 148893, upload-time = "2026-06-30T12:14:10.541Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ec/19d093e854b45e807fecfdd26105c266f43aeecc39c4dc97992a7074ad5a/tree_sitter-0.26.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6189c6c340c7384357711e3d92645e96bfb79f7a502f86de1ebdb23eb43f7dab", size = 140829, upload-time = "2026-06-30T12:14:11.626Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ee/87e74671ed63a837e7a1f17ab94aa3913871e033b27523d8e7b83d6f7ad0/tree_sitter-0.26.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ff2e0750b7daa722302838356d7b65e303829b7eb73c915df127ddba115e1d1", size = 639334, upload-time = "2026-06-30T12:14:12.836Z" }, + { url = "https://files.pythonhosted.org/packages/66/e7/f7e04cd9dff6b6ac0adf23922796fbc76accd4cf4bcda50542748d485679/tree_sitter-0.26.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7075ef857ef86f327dbb72d1e2574dda78db5754b3a1fca6506acd7fe5d561a7", size = 668102, upload-time = "2026-06-30T12:14:14.035Z" }, + { url = "https://files.pythonhosted.org/packages/d3/90/0bfb16b7894fea728c774a89d5af421a9368a2f913bbd4e8dcab7caaecfb/tree_sitter-0.26.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:26c996c1edfee86e977bb3f5462e74fcec0d0b0db1e85a3c475875763caa03be", size = 648560, upload-time = "2026-06-30T12:14:15.302Z" }, + { url = "https://files.pythonhosted.org/packages/cd/e6/0fe05ba396e9623b0ae40ccf34171336b8701ec8d7bd0ee9f5224d638665/tree_sitter-0.26.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00289bfe7978f3e0dc0ce69813a20fa9f44ea4c100b3ec62043e5eb74ccfc3a2", size = 665121, upload-time = "2026-06-30T12:14:16.403Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/a944b1ca35bed6068dc84a9967aaf3049d8cc0b7a36179eea8787270a6ab/tree_sitter-0.26.0-cp313-cp313-win_amd64.whl", hash = "sha256:93e220cab7e6a823efeb2046c49171427de92ef71c7c681c01820d14d8d3721f", size = 129615, upload-time = "2026-06-30T12:14:17.463Z" }, + { url = "https://files.pythonhosted.org/packages/09/ef/c7ca48293580d2249f36940c4eed5b4ddeb9ce75baf9a4ef30621987e0c7/tree_sitter-0.26.0-cp313-cp313-win_arm64.whl", hash = "sha256:b31a8195d2f224224c530ac814632d98c1dcc123d227442c07c736e86b70d564", size = 116525, upload-time = "2026-06-30T12:14:18.53Z" }, + { url = "https://files.pythonhosted.org/packages/c5/7a/4d84e6f6ae2c3e757490dd84de251712c31e293dfe31f28da1ec019cefa2/tree_sitter-0.26.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5a3c93a352b7e6f70f73e121bbfa2d0117ba7478bd51114ed35c91b0b78814fa", size = 148901, upload-time = "2026-06-30T12:14:19.452Z" }, + { url = "https://files.pythonhosted.org/packages/b0/d9/efe62ec65dc9d096e834d27b8c058127e2146e42ff3380b822a233f016a6/tree_sitter-0.26.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5fc2f41bf246ff2f70a9cc3690be35ec7580a4923151873d898c8bcb1a4503d3", size = 140805, upload-time = "2026-06-30T12:14:20.478Z" }, + { url = "https://files.pythonhosted.org/packages/c4/2c/c82326b7b97e3c485c18679883b16f89e5e913c639d3b219d3da70c9e67e/tree_sitter-0.26.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b8ea92a255c91671a7ec4625aba3ab7bb5220c423630ffbf83c45d7312abe084", size = 640586, upload-time = "2026-06-30T12:14:21.527Z" }, + { url = "https://files.pythonhosted.org/packages/e2/7a/f56e7d8282859452611024c7cbc623bfba5b24b8cb9b8f8bc88c5219fe9a/tree_sitter-0.26.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f665510f0fcf4636fb9696f1f7853bed7a3bd764b7bb0cb8494e619c14ed5a0c", size = 668300, upload-time = "2026-06-30T12:14:22.728Z" }, + { url = "https://files.pythonhosted.org/packages/91/51/240ee81b9d5e9ca0a6cb1528e8605ffa70ab58c89ce126631be96d3e4bae/tree_sitter-0.26.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:253df7ab82cc0a9d311cd65f06e9f99fb3eac55996ae9fc94da22f123a861b90", size = 649627, upload-time = "2026-06-30T12:14:23.819Z" }, + { url = "https://files.pythonhosted.org/packages/6a/54/760035cefedf9eb44f0f84c4ac22f1322e73155853e272576ee876336312/tree_sitter-0.26.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ff80d4833d330a73184a3ac5132abe93c575d2dea31975c6f15c0d21fef238aa", size = 664885, upload-time = "2026-06-30T12:14:25.064Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1b/0b36fe2a984ecedc4ce6aefd5d56447a6626a8e9b595c4e48658510ce8f8/tree_sitter-0.26.0-cp314-cp314-win_amd64.whl", hash = "sha256:a4033fecc8f606c7f2e8b8014d0057b74668a7f0152763606f7bc25c5f9ec64c", size = 132688, upload-time = "2026-06-30T12:14:26.106Z" }, + { url = "https://files.pythonhosted.org/packages/4d/74/ebc041a13fbf40144afdb0d4b447e48e0b4012ca866c63de8b48f801f0c1/tree_sitter-0.26.0-cp314-cp314-win_arm64.whl", hash = "sha256:823251c4b6725a7c03ed497a339135ede7ae4bdde75bb8be7ef5e305aeb4ff52", size = 120287, upload-time = "2026-06-30T12:14:26.991Z" }, +] + +[[package]] +name = "tree-sitter-c" +version = "0.24.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/c9/3834f3d9278251aea7312274971bc4c45b17aec2490fd4b884d93bd7019a/tree_sitter_c-0.24.2.tar.gz", hash = "sha256:1628584df0299b5a340aa63f8e67b6c97c91517f52fa7e7a4c557e40adb330a9", size = 228397, upload-time = "2026-04-22T08:06:14.491Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/c1/26ed17730ec2c17bedc1b673349e5e0a466c578e3eb0327c3b73cf52bf97/tree_sitter_c-0.24.2-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:4d4579a8b54f0a442f903d88d3304cab77cd5c2031d4015baa4f2f8e15d6dcb7", size = 81016, upload-time = "2026-04-22T08:06:07.208Z" }, + { url = "https://files.pythonhosted.org/packages/c1/1c/1140db75e7e375cda3c68792a33826c4fd40b5b98c3259d93c75f6c8368f/tree_sitter_c-0.24.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:97bc80a224d48215d4e6e6376bf30d114f4c317b8145ff1b02afe785d4ba7bdd", size = 86213, upload-time = "2026-04-22T08:06:08.136Z" }, + { url = "https://files.pythonhosted.org/packages/e9/8c/0dfb88d726f8821d1c4c36042f092be974a800afd734307a595b8604190c/tree_sitter_c-0.24.2-cp310-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5041ef67eb68ce6bc8bb0b1f8ef3a5585ce523dae0c7eec109ab0627dd75aede", size = 94264, upload-time = "2026-04-22T08:06:08.918Z" }, + { url = "https://files.pythonhosted.org/packages/87/78/47dc570e7aee6b0a1ecc2520b30639cc2b06003154c9ab0672d86bf720d5/tree_sitter_c-0.24.2-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c098bedcd5ac86ff93fa734d51d1dd86aed40fd5ed7d634c7af11380a0469969", size = 94560, upload-time = "2026-04-22T08:06:09.852Z" }, + { url = "https://files.pythonhosted.org/packages/29/37/75d59d3f74f4cfc00f04472917e933d8a9c9fdc6eff980ef9552e010e6aa/tree_sitter_c-0.24.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:82842c5a5f2acd93f4de10038c33ac179c8979defc39376f990348d6289e933b", size = 94023, upload-time = "2026-04-22T08:06:10.682Z" }, + { url = "https://files.pythonhosted.org/packages/64/57/8fc655d5a446a70a637e92b98bd2fdaab88bf5bb5b36076ac4add544808d/tree_sitter_c-0.24.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e2b42e8e22202c251f8629306f9321233542e07a6e01611b5fe83489272143eb", size = 94160, upload-time = "2026-04-22T08:06:11.497Z" }, + { url = "https://files.pythonhosted.org/packages/c1/f7/72a1d6b42dd31fd37e03ff67e7dc5ee572301499e6b216002b8dd42a1714/tree_sitter_c-0.24.2-cp310-abi3-win_amd64.whl", hash = "sha256:abb549225091f7b25df2dd3a0143ece6e208f7055d8bcb4700b41ee79b9ef1e1", size = 84669, upload-time = "2026-04-22T08:06:12.347Z" }, + { url = "https://files.pythonhosted.org/packages/e2/9d/7475d9ae8ef679aa36c7dfe6c903ab78e573651c68b6ef9862d6a3f994db/tree_sitter_c-0.24.2-cp310-abi3-win_arm64.whl", hash = "sha256:4a2f4371cd816cc3153458f69062135ebb2ea5f275ddd90494e5c823d778204a", size = 82956, upload-time = "2026-04-22T08:06:13.364Z" }, +] + +[[package]] +name = "tree-sitter-javascript" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/59/e0/e63103c72a9d3dfd89a31e02e660263ad84b7438e5f44ee82e443e65bbde/tree_sitter_javascript-0.25.0.tar.gz", hash = "sha256:329b5414874f0588a98f1c291f1b28138286617aa907746ffe55adfdcf963f38", size = 132338, upload-time = "2025-09-01T07:13:44.792Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/df/5106ac250cd03661ebc3cc75da6b3d9f6800a3606393a0122eca58038104/tree_sitter_javascript-0.25.0-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b70f887fb269d6e58c349d683f59fa647140c410cfe2bee44a883b20ec92e3dc", size = 64052, upload-time = "2025-09-01T07:13:36.865Z" }, + { url = "https://files.pythonhosted.org/packages/b1/8f/6b4b2bc90d8ab3955856ce852cc9d1e82c81d7ab9646385f0e75ffd5b5d3/tree_sitter_javascript-0.25.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:8264a996b8845cfce06965152a013b5d9cbb7d199bc3503e12b5682e62bb1de1", size = 66440, upload-time = "2025-09-01T07:13:37.962Z" }, + { url = "https://files.pythonhosted.org/packages/5f/c4/7da74ecdcd8a398f88bd003a87c65403b5fe0e958cdd43fbd5fd4a398fcf/tree_sitter_javascript-0.25.0-cp310-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9dc04ba91fc8583344e57c1f1ed5b2c97ecaaf47480011b92fbeab8dda96db75", size = 99728, upload-time = "2025-09-01T07:13:38.755Z" }, + { url = "https://files.pythonhosted.org/packages/96/c8/97da3af4796495e46421e9344738addb3602fa6426ea695be3fcbadbee37/tree_sitter_javascript-0.25.0-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:199d09985190852e0912da2b8d26c932159be314bc04952cf917ed0e4c633e6b", size = 106072, upload-time = "2025-09-01T07:13:39.798Z" }, + { url = "https://files.pythonhosted.org/packages/13/be/c964e8130be08cc9bd6627d845f0e4460945b158429d39510953bbcb8fcc/tree_sitter_javascript-0.25.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:dfcf789064c58dc13c0a4edb550acacfc6f0f280577f1e7a00de3e89fc7f8ddc", size = 104388, upload-time = "2025-09-01T07:13:40.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/89/9b773dee0f8961d1bb8d7baf0a204ab587618df19897c1ef260916f318ec/tree_sitter_javascript-0.25.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1b852d3aee8a36186dbcc32c798b11b4869f9b5041743b63b65c2ef793db7a54", size = 98377, upload-time = "2025-09-01T07:13:41.838Z" }, + { url = "https://files.pythonhosted.org/packages/3b/dc/d90cb1790f8cec9b4878d278ad9faf7c8f893189ce0f855304fd704fc274/tree_sitter_javascript-0.25.0-cp310-abi3-win_amd64.whl", hash = "sha256:e5ed840f5bd4a3f0272e441d19429b26eedc257abe5574c8546da6b556865e3c", size = 62975, upload-time = "2025-09-01T07:13:42.828Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1f/f9eba1038b7d4394410f3c0a6ec2122b590cd7acb03f196e52fa57ebbe72/tree_sitter_javascript-0.25.0-cp310-abi3-win_arm64.whl", hash = "sha256:622a69d677aa7f6ee2931d8c77c981a33f0ebb6d275aa9d43d3397c879a9bb0b", size = 61668, upload-time = "2025-09-01T07:13:43.803Z" }, +] + +[[package]] +name = "tree-sitter-python" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b8/8b/c992ff0e768cb6768d5c96234579bf8842b3a633db641455d86dd30d5dac/tree_sitter_python-0.25.0.tar.gz", hash = "sha256:b13e090f725f5b9c86aa455a268553c65cadf325471ad5b65cd29cac8a1a68ac", size = 159845, upload-time = "2025-09-11T06:47:58.159Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/64/a4e503c78a4eb3ac46d8e72a29c1b1237fa85238d8e972b063e0751f5a94/tree_sitter_python-0.25.0-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:14a79a47ddef72f987d5a2c122d148a812169d7484ff5c75a3db9609d419f361", size = 73790, upload-time = "2025-09-11T06:47:47.652Z" }, + { url = "https://files.pythonhosted.org/packages/e6/1d/60d8c2a0cc63d6ec4ba4e99ce61b802d2e39ef9db799bdf2a8f932a6cd4b/tree_sitter_python-0.25.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:480c21dbd995b7fe44813e741d71fed10ba695e7caab627fb034e3828469d762", size = 76691, upload-time = "2025-09-11T06:47:49.038Z" }, + { url = "https://files.pythonhosted.org/packages/aa/cb/d9b0b67d037922d60cbe0359e0c86457c2da721bc714381a63e2c8e35eba/tree_sitter_python-0.25.0-cp310-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:86f118e5eecad616ecdb81d171a36dde9bef5a0b21ed71ea9c3e390813c3baf5", size = 108133, upload-time = "2025-09-11T06:47:50.499Z" }, + { url = "https://files.pythonhosted.org/packages/40/bd/bf4787f57e6b2860f3f1c8c62f045b39fb32d6bac4b53d7a9e66de968440/tree_sitter_python-0.25.0-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be71650ca2b93b6e9649e5d65c6811aad87a7614c8c1003246b303f6b150f61b", size = 110603, upload-time = "2025-09-11T06:47:51.985Z" }, + { url = "https://files.pythonhosted.org/packages/5d/25/feff09f5c2f32484fbce15db8b49455c7572346ce61a699a41972dea7318/tree_sitter_python-0.25.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e6d5b5799628cc0f24691ab2a172a8e676f668fe90dc60468bee14084a35c16d", size = 108998, upload-time = "2025-09-11T06:47:53.046Z" }, + { url = "https://files.pythonhosted.org/packages/75/69/4946da3d6c0df316ccb938316ce007fb565d08f89d02d854f2d308f0309f/tree_sitter_python-0.25.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:71959832fc5d9642e52c11f2f7d79ae520b461e63334927e93ca46cd61cd9683", size = 107268, upload-time = "2025-09-11T06:47:54.388Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a2/996fc2dfa1076dc460d3e2f3c75974ea4b8f02f6bc925383aaae519920e8/tree_sitter_python-0.25.0-cp310-abi3-win_amd64.whl", hash = "sha256:9bcde33f18792de54ee579b00e1b4fe186b7926825444766f849bf7181793a76", size = 76073, upload-time = "2025-09-11T06:47:55.773Z" }, + { url = "https://files.pythonhosted.org/packages/07/19/4b5569d9b1ebebb5907d11554a96ef3fa09364a30fcfabeff587495b512f/tree_sitter_python-0.25.0-cp310-abi3-win_arm64.whl", hash = "sha256:0fbf6a3774ad7e89ee891851204c2e2c47e12b63a5edbe2e9156997731c128bb", size = 74169, upload-time = "2025-09-11T06:47:56.747Z" }, +] + +[[package]] +name = "tree-sitter-typescript" +version = "0.23.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/fc/bb52958f7e399250aee093751e9373a6311cadbe76b6e0d109b853757f35/tree_sitter_typescript-0.23.2.tar.gz", hash = "sha256:7b167b5827c882261cb7a50dfa0fb567975f9b315e87ed87ad0a0a3aedb3834d", size = 773053, upload-time = "2024-11-11T02:36:11.396Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/95/4c00680866280e008e81dd621fd4d3f54aa3dad1b76b857a19da1b2cc426/tree_sitter_typescript-0.23.2-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3cd752d70d8e5371fdac6a9a4df9d8924b63b6998d268586f7d374c9fba2a478", size = 286677, upload-time = "2024-11-11T02:35:58.839Z" }, + { url = "https://files.pythonhosted.org/packages/8f/2f/1f36fda564518d84593f2740d5905ac127d590baf5c5753cef2a88a89c15/tree_sitter_typescript-0.23.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:c7cc1b0ff5d91bac863b0e38b1578d5505e718156c9db577c8baea2557f66de8", size = 302008, upload-time = "2024-11-11T02:36:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/96/2d/975c2dad292aa9994f982eb0b69cc6fda0223e4b6c4ea714550477d8ec3a/tree_sitter_typescript-0.23.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b1eed5b0b3a8134e86126b00b743d667ec27c63fc9de1b7bb23168803879e31", size = 351987, upload-time = "2024-11-11T02:36:02.669Z" }, + { url = "https://files.pythonhosted.org/packages/49/d1/a71c36da6e2b8a4ed5e2970819b86ef13ba77ac40d9e333cb17df6a2c5db/tree_sitter_typescript-0.23.2-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e96d36b85bcacdeb8ff5c2618d75593ef12ebaf1b4eace3477e2bdb2abb1752c", size = 344960, upload-time = "2024-11-11T02:36:04.443Z" }, + { url = "https://files.pythonhosted.org/packages/7f/cb/f57b149d7beed1a85b8266d0c60ebe4c46e79c9ba56bc17b898e17daf88e/tree_sitter_typescript-0.23.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8d4f0f9bcb61ad7b7509d49a1565ff2cc363863644a234e1e0fe10960e55aea0", size = 340245, upload-time = "2024-11-11T02:36:06.473Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ab/dd84f0e2337296a5f09749f7b5483215d75c8fa9e33738522e5ed81f7254/tree_sitter_typescript-0.23.2-cp39-abi3-win_amd64.whl", hash = "sha256:3f730b66396bc3e11811e4465c41ee45d9e9edd6de355a58bbbc49fa770da8f9", size = 278015, upload-time = "2024-11-11T02:36:07.631Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e4/81f9a935789233cf412a0ed5fe04c883841d2c8fb0b7e075958a35c65032/tree_sitter_typescript-0.23.2-cp39-abi3-win_arm64.whl", hash = "sha256:05db58f70b95ef0ea126db5560f3775692f609589ed6f8dd0af84b7f19f1cbb7", size = 274052, upload-time = "2024-11-11T02:36:09.514Z" }, +] + [[package]] name = "ty" version = "0.0.46" From 15c619581a203af56348607e5c2ab72b165b7d7c Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Tue, 8 Sep 2026 22:23:27 -0700 Subject: [PATCH 14/16] build(server): take the lance extension from PyPI instead of fetching it in the image duckdb-extension-lance ships lance.duckdb_extension inside its wheel and pins duckdb to the release it was built for, so the resolver now owns the version match that the Dockerfile ARG and the --check script used to enforce. The image no longer fetches, verifies or stages the extension; a consumer LOADs it by path from site-packages. Drop the two build scripts and the CI smoke step with them. The only check they made that no Python test covers, tessdata being present at TESSDATA_PREFIX, is now a one-line assertion in the Dockerfile. --- .../extralit-server.build-docker-images.yml | 4 - extralit-server/docker/server/.dockerignore | 4 +- extralit-server/docker/server/Dockerfile | 22 +--- .../server/scripts/install_lance_extension.py | 124 ------------------ .../server/scripts/smoke_retrieval_deps.py | 70 ---------- extralit-server/pyproject.toml | 2 + extralit-server/uv.lock | 90 +++++++------ 7 files changed, 59 insertions(+), 257 deletions(-) delete mode 100644 extralit-server/docker/server/scripts/install_lance_extension.py delete mode 100644 extralit-server/docker/server/scripts/smoke_retrieval_deps.py diff --git a/.github/workflows/extralit-server.build-docker-images.yml b/.github/workflows/extralit-server.build-docker-images.yml index ca3e3d458..3ce6a58e1 100644 --- a/.github/workflows/extralit-server.build-docker-images.yml +++ b/.github/workflows/extralit-server.build-docker-images.yml @@ -125,10 +125,6 @@ jobs: run: | docker run --rm ${{ env.SERVER_DOCKER_IMAGE }}:${{ env.IMAGE_TAG }} python -m extralit_server start --help - - name: Check the retrieval payloads are baked into the image - run: | - docker run --rm ${{ env.SERVER_DOCKER_IMAGE }}:${{ env.IMAGE_TAG }} python smoke_retrieval_deps.py - - name: Push latest `extralit-server` image if: ${{ env.PUBLISH_LATEST == 'true' }} uses: docker/build-push-action@v5 diff --git a/extralit-server/docker/server/.dockerignore b/extralit-server/docker/server/.dockerignore index ee0676668..0c443d341 100644 --- a/extralit-server/docker/server/.dockerignore +++ b/extralit-server/docker/server/.dockerignore @@ -1,6 +1,4 @@ -# Allowlist: the context is a wheel and two scripts, and should stay that way however much -# stray build output lands in this directory. +# Allowlist: the context is a wheel and one script, whatever stray build output lands here. * -!scripts/*.py !scripts/*.sh !dist/*.whl diff --git a/extralit-server/docker/server/Dockerfile b/extralit-server/docker/server/Dockerfile index 2f640904d..e1bcc51bd 100644 --- a/extralit-server/docker/server/Dockerfile +++ b/extralit-server/docker/server/Dockerfile @@ -1,14 +1,3 @@ -# The duckdb the venv resolves. Pinned so the 231 MB extension fetch can sit in a stage that does -# not depend on the application wheel; the `--check` below fails the build if the pin ever drifts. -ARG DUCKDB_VERSION=1.5.5 - -# Fetched in its own stage, ahead of anything that changes per build: DuckDB looks extensions up -# under a version-named directory, so this layer is identical until duckdb itself moves. -FROM python:3.12-slim AS lance -ARG DUCKDB_VERSION -COPY scripts/install_lance_extension.py /tmp/ -RUN python /tmp/install_lance_extension.py --duckdb-home /lance --duckdb-version "$DUCKDB_VERSION" - FROM python:3.12-slim AS builder COPY --from=ghcr.io/astral-sh/uv:0.12.6 /uv /uvx /bin/ @@ -42,7 +31,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \ for wheel in /packages/*.whl; do uv pip install "$wheel"[postgresql]; done FROM python:3.12-slim -ARG DUCKDB_VERSION # Environment Variables ENV USERNAME="" @@ -69,11 +57,11 @@ VOLUME $EXTRALIT_HOME_PATH # liteparse would otherwise download the language data on the first scanned page it meets. ENV TESSDATA_PREFIX=/usr/share/tesseract-ocr/5/tessdata +RUN test -f "$TESSDATA_PREFIX/eng.traineddata" # Layers below are ordered by how often they change, because a rebuilt layer rebuilds every layer -# under it. DuckDB resolves extensions per user home, hence extralit's rather than root's. -COPY --from=lance --chown=extralit:extralit /lance /home/extralit/.duckdb -COPY --chmod=0755 scripts/start_extralit_server.sh scripts/smoke_retrieval_deps.py scripts/install_lance_extension.py /home/extralit/ +# under it. +COPY --chmod=0755 scripts/start_extralit_server.sh /home/extralit/ # Destination folder must be the same as the builder one. Otherwise installed script won't work (since the installation fixes the path inside the script) COPY --chown=extralit:extralit --from=builder /opt/venv /opt/venv @@ -84,10 +72,6 @@ WORKDIR /home/extralit USER extralit -# Fails the build rather than shipping an image whose first hybrid search cannot load the -# extension. Reads metadata only, so it survives the emulated arch of a release build. -RUN python install_lance_extension.py --check --duckdb-version "$DUCKDB_VERSION" - # Exposing ports EXPOSE 6900 diff --git a/extralit-server/docker/server/scripts/install_lance_extension.py b/extralit-server/docker/server/scripts/install_lance_extension.py deleted file mode 100644 index 65b02a5bc..000000000 --- a/extralit-server/docker/server/scripts/install_lance_extension.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Bake DuckDB's `lance` extension into the image, for this stage's target architecture. - - python install_lance_extension.py --duckdb-home /lance --duckdb-version 1.5.5 - python install_lance_extension.py --check --duckdb-version 1.5.5 - -`INSTALL lance` would be the obvious way to fetch it, but a release builds the non-native arch -under QEMU, where importing duckdb segfaults. Nothing here loads the native module: the payload -is a plain download and `--check` reads package metadata, so both survive emulation. - -The version is passed in rather than derived so the fetch can live in a stage that does not -depend on the application wheel — otherwise a one-line code change re-downloads 231 MB. `--check` -is what keeps that pin honest: it fails the build if the venv resolved a different duckdb. - -This extension is native code that the server later loads into its own process, so the payload -is fetched over TLS and checked against a pinned digest before it is written. An unpinned -version fails the build rather than trusting whatever the network returned. -""" - -from __future__ import annotations - -import argparse -import gzip -import hashlib -import platform -import urllib.request -from importlib.metadata import version -from pathlib import Path - -REPOSITORY = "https://extensions.duckdb.org" -# The CDN 403s the default Python-urllib agent; any identifiable one is served. -USER_AGENT = "extralit-server image build" -PLATFORMS = {"aarch64": "linux_arm64", "x86_64": "linux_amd64"} - -#: sha256 of the *decompressed* extension, per duckdb version and target platform. Bumping -#: DUCKDB_VERSION means adding an entry here; see `--digest` for how to produce one. -DIGESTS = { - "1.5.5": { - "linux_amd64": "a8b1463e8541a960859b05c39096a60a3c777d12fd63d9867ce62bb251ab2a1a", - "linux_arm64": "a710b8c2453e996ff810c718ccc9b26253a1cf6c1b6687fad0dd805f8878e2c9", - }, -} - - -def target_platform() -> str: - machine = platform.machine() - try: - return PLATFORMS[machine] - except KeyError: - raise SystemExit(f"no DuckDB extension platform for {machine}") from None - - -def expected_digest(duckdb_version: str, target: str) -> str: - try: - return DIGESTS[duckdb_version][target] - except KeyError: - raise SystemExit( - f"no pinned sha256 for the lance extension at duckdb {duckdb_version} on {target}. " - f"This build will not load an unverified native extension into the server. Obtain the " - f"digest, confirm it independently of this download, and add it to DIGESTS in " - f"{Path(__file__).name} (`--digest` prints what the repository is currently serving)." - ) from None - - -def download(duckdb_version: str, target: str) -> bytes: - url = f"{REPOSITORY}/v{duckdb_version}/{target}/lance.duckdb_extension.gz" - request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) - with urllib.request.urlopen(request) as response: - return gzip.decompress(response.read()) - - -def check(expected: str) -> None: - installed = version("duckdb") - if installed != expected: - raise SystemExit( - f"the image bakes the lance extension for duckdb {expected}, but the venv resolved " - f"{installed}. DuckDB looks extensions up under a version-named directory, so this " - f"would ship an image whose first hybrid search fails. Set DUCKDB_VERSION={installed} " - f"in the Dockerfile." - ) - - -def fetch(duckdb_version: str, duckdb_home: Path) -> None: - target = target_platform() - expected = expected_digest(duckdb_version, target) - - payload = download(duckdb_version, target) - digest = hashlib.sha256(payload).hexdigest() - if digest != expected: - # Nothing is written: the bytes are not what this image is pinned to. - raise SystemExit( - f"the lance extension served for duckdb {duckdb_version} on {target} has sha256 " - f"{digest}, but this image pins {expected}. Refusing to bake an unverified native " - f"extension." - ) - - destination = duckdb_home / "extensions" / f"v{duckdb_version}" / target - destination.mkdir(parents=True, exist_ok=True) - written = destination / "lance.duckdb_extension" - written.write_bytes(payload) - print(f"wrote {written} ({len(payload)} bytes, sha256 {digest})") - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--duckdb-version", help="defaults to the installed duckdb") - parser.add_argument("--duckdb-home", type=Path, default=Path.home() / ".duckdb") - parser.add_argument("--check", action="store_true", help="assert the venv agrees with the pin") - parser.add_argument("--digest", action="store_true", help="print the served digest, for pinning") - args = parser.parse_args() - - duckdb_version = args.duckdb_version or version("duckdb") - if args.check: - check(duckdb_version) - print(f"ok: venv duckdb matches the baked lance extension ({duckdb_version})") - return - if args.digest: - target = target_platform() - print(f'"{target}": "{hashlib.sha256(download(duckdb_version, target)).hexdigest()}",') - return - fetch(duckdb_version, args.duckdb_home) - - -if __name__ == "__main__": - main() diff --git a/extralit-server/docker/server/scripts/smoke_retrieval_deps.py b/extralit-server/docker/server/scripts/smoke_retrieval_deps.py deleted file mode 100644 index 5bc327213..000000000 --- a/extralit-server/docker/server/scripts/smoke_retrieval_deps.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Assert the retrieval pipeline's non-Python payloads are present in this environment. - - docker run --rm extralit/extralit-server:latest python smoke_retrieval_deps.py - -Each of these fails late and confusingly in production — a first hybrid search that cannot -reach the DuckDB extension repository, or a first scanned page that finds no language data — -so the image build runs this instead of discovering them under load. -""" - -from __future__ import annotations - -import os -import sys -from pathlib import Path - - -def check_lance_extension() -> str: - import duckdb - - connection = duckdb.connect() - # No INSTALL: the point is that the extension is already on disk. - connection.execute("LOAD lance") - functions = connection.execute( - "select function_name from duckdb_functions() where function_name in ('lance_fts', 'lance_vector_search')" - ).fetchall() - missing = {"lance_fts", "lance_vector_search"} - {name for (name,) in functions} - if missing: - raise RuntimeError(f"the lance extension loaded without {sorted(missing)}") - return f"duckdb {duckdb.__version__} + lance extension" - - -def check_liteparse() -> str: - import liteparse - - parser = liteparse.LiteParse(extract_blocks=True, quiet=True) - parser.close() - return f"liteparse {liteparse.__version__}" - - -def check_tessdata() -> str: - prefix = os.environ.get("TESSDATA_PREFIX") - if not prefix: - raise RuntimeError("TESSDATA_PREFIX is unset, so OCR would download language data on first use") - if not (Path(prefix) / "eng.traineddata").is_file(): - raise RuntimeError(f"no eng.traineddata under {prefix}") - return f"tessdata at {prefix}" - - -def check_docling_chunker() -> str: - from importlib.metadata import version - - from docling_core.transforms.chunker.hybrid_chunker import HybridChunker # noqa: F401 - - return f"docling-core {version('docling-core')}" - - -def main() -> int: - failures = 0 - for check in (check_lance_extension, check_liteparse, check_tessdata, check_docling_chunker): - name = check.__name__.removeprefix("check_") - try: - print(f"ok {name}: {check()}") - except Exception as error: - failures += 1 - print(f"FAIL {name}: {type(error).__name__}: {error}", file=sys.stderr) - return 1 if failures else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/extralit-server/pyproject.toml b/extralit-server/pyproject.toml index 2af2485e3..0097d0fef 100644 --- a/extralit-server/pyproject.toml +++ b/extralit-server/pyproject.toml @@ -77,6 +77,8 @@ dependencies = [ "lancedb>=0.37.1", "pylance>=10.0.0", "duckdb>=1.5.4", + # ships lance.duckdb_extension in the wheel and pins duckdb to the release it was built for + "duckdb-extension-lance>=1.5.5", "docling-core>=2.91.0,<3.0.0", "pyarrow>=23.0.1", "pdf-inspector>=1.14.2", diff --git a/extralit-server/uv.lock b/extralit-server/uv.lock index d3e9900ee..1247901ba 100644 --- a/extralit-server/uv.lock +++ b/extralit-server/uv.lock @@ -924,44 +924,58 @@ wheels = [ [[package]] name = "duckdb" -version = "1.5.4" +version = "1.5.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/19/e57151753576373c6696a12022648546cca6038e8833fda2908ee2342d9b/duckdb-1.5.5.tar.gz", hash = "sha256:72f33ee57ca7595b23957671a2cc7f7fe2be0ecc2d68f63abedcfcaa3a5c1238", size = 18066741, upload-time = "2026-07-22T10:55:17.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/d4/298acf9331a80b3ce6ac64dd940e7e13f4058fb69d18914445f02e3c7bfe/duckdb-1.5.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3b805507f88171b428b21c966c30e9a3d54e30b24528918a44ed0032542bc26f", size = 32702934, upload-time = "2026-07-22T10:53:19.069Z" }, + { url = "https://files.pythonhosted.org/packages/d5/90/c489fb63d64b2e7ee109ce8460bdede003a0f256e5b41a03a2a1c4764058/duckdb-1.5.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b08e19cc856220d8a26fa62abc2264b349aff67255e9373c6a3f607addd56dc6", size = 17343604, upload-time = "2026-07-22T10:53:22.767Z" }, + { url = "https://files.pythonhosted.org/packages/0b/27/effa80a15b1f0c61c235622f797868485359e8c9ad6a8e358e7a0c479151/duckdb-1.5.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a17e6a922e42a5c06ed2353fe78c5dff2610f6632d603836f9606ad0bf754079", size = 15488179, upload-time = "2026-07-22T10:53:25.945Z" }, + { url = "https://files.pythonhosted.org/packages/5d/07/21212345c8d24ba62dceaa20be3b21f5c46f1510b1b42ce93bb058afe0c4/duckdb-1.5.5-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1bdc38922c365c37720149f90d90b1e9823eb82dad6830855b5f87537fa6fc0c", size = 19367323, upload-time = "2026-07-22T10:53:30.23Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d0/10371ae875fb4b5ef61bb892743b4b2e90c512b371fdf29317deb744857d/duckdb-1.5.5-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e238060db5ca59879882a6e9b015e2c65d5c64ddf281ba1d7a9a2033764152cf", size = 21476568, upload-time = "2026-07-22T10:53:33.486Z" }, + { url = "https://files.pythonhosted.org/packages/b2/35/09568ce617dd7bc0757b3d7b6a981660b9e4f0b7594de8ed776755eae740/duckdb-1.5.5-cp310-cp310-win_amd64.whl", hash = "sha256:4acc72798ba1885a9c17d1242903d2cd502f13b1271c7677f7cab25d8578eceb", size = 13156129, upload-time = "2026-07-22T10:53:37.55Z" }, + { url = "https://files.pythonhosted.org/packages/9c/c2/b62ec24d57bb8df4e24b0b58f7f8facb32f5fdb9f1895aed9e9fcdded168/duckdb-1.5.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1b543841b0ae18a9c982345cfa3987e9c065d3a4b0f067daa473d92d1e65f528", size = 32708371, upload-time = "2026-07-22T10:53:41.642Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ce/769171ba45f0b73632dc3bc3108d891e81dd6c6bbfba630a34a75b4dcc0f/duckdb-1.5.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a925d06c2a4c3b64553d6cc1aced5028d376d4479bed689a7d47e9b1dccd80a", size = 17343979, upload-time = "2026-07-22T10:53:44.951Z" }, + { url = "https://files.pythonhosted.org/packages/46/59/a8e3384ee916e00d5dcf985194c1511d61978540778a1e96fa47f9fb3e0d/duckdb-1.5.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0c42757cb34722144bd4dfb94b6f336339e7b2468f6813fa7fa9a319ba07bab4", size = 15493704, upload-time = "2026-07-22T10:53:47.912Z" }, + { url = "https://files.pythonhosted.org/packages/6f/1d/9840179c2607b90523a2884a129c4d4e6dbdc1178ba62a976c1043beba88/duckdb-1.5.5-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e72f9e1a4f90a5c8483ad4d540e495bf0834ba61c360b52499a573d7ed62a3f", size = 19366574, upload-time = "2026-07-22T10:53:51.876Z" }, + { url = "https://files.pythonhosted.org/packages/b5/55/f9641a4eebcc2f4df631287d6c3b9ed2eea3b92644f93acbad825e3972b6/duckdb-1.5.5-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9b6f86ed85d4ef5e0211eaebf75d057bd8bb520bba438a95dd0f4e42234bbfe", size = 21477952, upload-time = "2026-07-22T10:53:55.575Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3a/07c3556e37a5c97b95917b029c8fdde4a25fbd76a660bacdac195cf20dcb/duckdb-1.5.5-cp311-cp311-win_amd64.whl", hash = "sha256:9f4287f97ccf0c1f3d471e7115be2b067cbf99627e2d34bffd462dd64703cddc", size = 13156986, upload-time = "2026-07-22T10:53:58.823Z" }, + { url = "https://files.pythonhosted.org/packages/4f/ff/07b48eef2078ca033847e9caa46cc7633b714c5f91ad1ce091c8ca89d792/duckdb-1.5.5-cp311-cp311-win_arm64.whl", hash = "sha256:179633a3fc6296c75d57c69c1e239fa9e5cdcb670fd1dbff88a02663f932905c", size = 14001317, upload-time = "2026-07-22T10:54:01.724Z" }, + { url = "https://files.pythonhosted.org/packages/d6/40/2e05d324400fdaa5656c9f48d6298da421cb034d85e509fa0e6e325cf04b/duckdb-1.5.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d4dd65f8941a604b947e0b9b4b4f7165988e29a23ec0b69b4038520956d9933e", size = 32753858, upload-time = "2026-07-22T10:54:05.514Z" }, + { url = "https://files.pythonhosted.org/packages/79/15/5ceb58ffb5bb8a62b3fd7abb39c41467cdf94850ece02e6d88664dfc75ce/duckdb-1.5.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:33db46679b071f108d57139493dee2d37e1f5efcf5c5c039c2969eed11a6c8a7", size = 17368293, upload-time = "2026-07-22T10:54:09.139Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5c/bf02da0b354fe83cca4f95a4fbf762181af466f7d551ab2a093f7698882a/duckdb-1.5.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f0b88535a5d86fdd63dba6ea02ab68c003dfb9e4892b11256ef24c4da208baae", size = 15509131, upload-time = "2026-07-22T10:54:12.228Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a9/5f1f09da421d8e930e0b063d11c1b3f90363f40ede74438cd188afdd13a2/duckdb-1.5.5-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f316eae2323d9a851883fdf2dee91c1f9efe251ab33e14a2272f82a913422ed6", size = 19391959, upload-time = "2026-07-22T10:54:15.551Z" }, + { url = "https://files.pythonhosted.org/packages/4f/98/6549769f158126fa64fd6c1ac2eb59a18282146c939867a3eb31b7c1db07/duckdb-1.5.5-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7a6d2d11859d82a936ebdcb30ce3d8a1cbb3e990bff05c12abb9b54c44fa7bd1", size = 21510909, upload-time = "2026-07-22T10:54:19.681Z" }, + { url = "https://files.pythonhosted.org/packages/af/b7/5753b41d3124838f868f9f523362812d9fc45409e9e4dd70dcbb0a25826e/duckdb-1.5.5-cp312-cp312-win_amd64.whl", hash = "sha256:ddfbdb096c11d51ee22492397d342c90a82e62c5d09961477895934d0a25372f", size = 13168544, upload-time = "2026-07-22T10:54:22.789Z" }, + { url = "https://files.pythonhosted.org/packages/5c/28/44b679c7d46245f8398feae7edac959d1b83d4eb143e25b3fce0630b78bd/duckdb-1.5.5-cp312-cp312-win_arm64.whl", hash = "sha256:2725d2b9ace3a4e75d72fc5a239f6a44b502c580edadb8fb2676db772c5f9282", size = 13988684, upload-time = "2026-07-22T10:54:26.003Z" }, + { url = "https://files.pythonhosted.org/packages/47/37/4a38116e7700720fd152c666292214fd3abdf916496991296d8d1f66efbf/duckdb-1.5.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:cd98829b67788609017e65c761bd42a5dd0f9129441bed8bda4d6881ccf819f0", size = 32754294, upload-time = "2026-07-22T10:54:29.822Z" }, + { url = "https://files.pythonhosted.org/packages/66/42/7d392f1ba1eee0eaf4ab4c8c7a604bfe3536cd63f979cf5c98798664f807/duckdb-1.5.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:feead93c56679b79592d437c62975d39cb67adedffa7592c763baf8160ac7366", size = 17368211, upload-time = "2026-07-22T10:54:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a5/0a6f4fa60562faa615e55e15bd1953a2f2b17a8edd8105e5cda215e43457/duckdb-1.5.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:49c963d9469373d7aba8d750d9ea565ab823e94166efed953f184dd9b169b98c", size = 15509136, upload-time = "2026-07-22T10:54:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/e4/cb/023c89f51978545b9fab318581bba0c457a58e7530d2d933e54ae7d8647c/duckdb-1.5.5-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a736217825461732b5442d05a220f3da2e23a0dae114efbf08c9bf171b53098a", size = 19392147, upload-time = "2026-07-22T10:54:39.551Z" }, + { url = "https://files.pythonhosted.org/packages/3e/c5/41bef391fb8b23dbc133c9f2ba016e7a7a8124513d2cc1b430f1897d87e4/duckdb-1.5.5-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:078e6a60dd8eedde5832f45422ca5c4a6b8c837aeabd8a56ca0b7d933f588053", size = 21511060, upload-time = "2026-07-22T10:54:42.788Z" }, + { url = "https://files.pythonhosted.org/packages/07/9f/c44dfc1f924ac29b3252dc1b91393c01d009dbfe9f8ed33f10b986151bd1/duckdb-1.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:6826504277dba513c0c5d71d828456c94d729c9d2482f94b2e289f90a9167e28", size = 13168028, upload-time = "2026-07-22T10:54:46.127Z" }, + { url = "https://files.pythonhosted.org/packages/ca/88/591384b2cd59abddd6f5dc175e60374f9abae6064429f0c4402854c10f44/duckdb-1.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:baa9c5702002fabb559ded2a39008f9f421fcbc7237d388b8213eff1e08858de", size = 13989955, upload-time = "2026-07-22T10:54:49.262Z" }, + { url = "https://files.pythonhosted.org/packages/3e/56/12c65bfa2d2605b81981b264788891bcf11ec72227889554cead5d8d13b9/duckdb-1.5.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8e6413dd40facb7b8ab21bd844450cd8f549b29e138635be9cf090ef4d2049e2", size = 32761946, upload-time = "2026-07-22T10:54:53.412Z" }, + { url = "https://files.pythonhosted.org/packages/b9/46/682ce155f17e0d2822d4f13ee3db9ca4b5b7c2da61b841b2629035e1f4bc/duckdb-1.5.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:64078acfd16541132ac6e191eb81b2845554444a0305cc1aa581ba107e514aa8", size = 17375069, upload-time = "2026-07-22T10:54:57.269Z" }, + { url = "https://files.pythonhosted.org/packages/39/ce/a24bcbd3289c8f305a430759c5fc12242740b4af3e17f7593f3a34e333d2/duckdb-1.5.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8c11775cc99a447618d5f1840126db17f2652f3eae05529df4f81f40e2df7151", size = 15519791, upload-time = "2026-07-22T10:55:00.681Z" }, + { url = "https://files.pythonhosted.org/packages/d9/76/3a01afbc615c1d418c0de58a6b68ac5ce2a8563232c0464bfbc2ce552398/duckdb-1.5.5-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77bbc1e6ba12e1e06f9020117bdf848627ecfdf36f907550e62e008e6109dece", size = 19398251, upload-time = "2026-07-22T10:55:04.168Z" }, + { url = "https://files.pythonhosted.org/packages/a1/43/3a5e81d1728f4d234c79bfe385808ee7c04834f7c37a4b5c257459c25614/duckdb-1.5.5-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fbf0f2d48b43c6c304d00463b463c27ead6c4b01c3c1816b750f728decf71afe", size = 21513851, upload-time = "2026-07-22T10:55:07.864Z" }, + { url = "https://files.pythonhosted.org/packages/91/41/fc7c829172c60ca22485251eab285f4f1a0d87b486a024c726f21471d86e/duckdb-1.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:9dc826c4b50e64f6c4e4d07a3a9cb075ef70ba3899dc43ec5493dc3d7b04b353", size = 13691858, upload-time = "2026-07-22T10:55:11.181Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2c/95d9216b79e9273689d7ebce125a54503ed0c9bd7da931f0265888e99779/duckdb-1.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:63e48d4b74b15aeacd688976432a7225163df8c226eddeb8536bba2d4d4ff433", size = 14470180, upload-time = "2026-07-22T10:55:14.445Z" }, +] + +[[package]] +name = "duckdb-extension-lance" +version = "1.5.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/29/9bad86ed7aa812d8c822a27c15c355b6d5423b991feeec86ed18027b6daa/duckdb-1.5.4.tar.gz", hash = "sha256:f9e32f1cdd106793d79d190186bed9e75289d51e68bd9174e47c04bffedeab6f", size = 18046634, upload-time = "2026-06-17T10:48:52.499Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/ee/69340af74a3aa21838f14c16a0dd3e58461896ccba41f6bc7f0a01536e23/duckdb-1.5.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3ddd9533ce80f9b851bdd6276960a9286166514a9ceca43d5bc2f0d5842c490d", size = 32656177, upload-time = "2026-06-17T10:47:31.044Z" }, - { url = "https://files.pythonhosted.org/packages/73/18/9da267ade389d4e7e533ac0c77b3a7041513a66efab93beb84f27627b0b8/duckdb-1.5.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:360f2542d09759c3739400f8b787e29b43ba0da665c21756216291458bf6fc59", size = 17318966, upload-time = "2026-06-17T10:47:33.688Z" }, - { url = "https://files.pythonhosted.org/packages/76/b4/ad73c1a396288e443b18af50819448060b318c1e933305167c1d7f98a507/duckdb-1.5.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0cf932055061e544d3fa27cc6c147da25f3f681ee5980157fb55e77d6c2d9c63", size = 15467572, upload-time = "2026-06-17T10:47:36.071Z" }, - { url = "https://files.pythonhosted.org/packages/6f/06/2c52ce3b97c3f21111f3c98a2121ed002e33f86488f55098a37825af6d4a/duckdb-1.5.4-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c2d58d39f5e65419cdc27e3875cba4a729a3bbf6bf4016aefb4a2a65335a1d42", size = 19344044, upload-time = "2026-06-17T10:47:38.174Z" }, - { url = "https://files.pythonhosted.org/packages/5a/7d/5c0cc66fb90a1b14474eebb5ab535eacc51cb20b0e45358348b51c07abc9/duckdb-1.5.4-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9a10dc40469b9c0e458625d2a8359461a982c6151bb53ff259fea00c4695ad4", size = 21448215, upload-time = "2026-06-17T10:47:40.617Z" }, - { url = "https://files.pythonhosted.org/packages/3c/2c/16c3ea201855cdfb7dde52ea3678d0536861cb485ffad46cf345436d658d/duckdb-1.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:3565550adbf160ef7a2ee3395470570182f11233983ad818bd7d5f9e349f92b2", size = 13132138, upload-time = "2026-06-17T10:47:43.085Z" }, - { url = "https://files.pythonhosted.org/packages/56/bb/7921dabd50daef3969f14cd8a5a14c24eee337db7914a462f2defa8add92/duckdb-1.5.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3fb41d9cfccb7e44511eeeed263ae98143ca63bdb1ef84631ba637c314efa1b5", size = 32663142, upload-time = "2026-06-17T10:47:45.471Z" }, - { url = "https://files.pythonhosted.org/packages/a6/83/2137765eaba6a9aefe3bb9848ddaac7407fe3ba19b292f98b31f3b7ab27f/duckdb-1.5.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8ba7b666bc9c78d6a930ee9f469024149f0c6a23fb7d2c3418aad6774339bec0", size = 17321485, upload-time = "2026-06-17T10:47:47.778Z" }, - { url = "https://files.pythonhosted.org/packages/0e/b2/a02c1ee43fd7e8cf1fc2e3d377f3dcf9d4a3e58a4549557516e1866ff0da/duckdb-1.5.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9d9e6817fcbc09d2605a2c8c041ac7824d738d917c35a4d427e977647e1d7944", size = 15470820, upload-time = "2026-06-17T10:47:49.977Z" }, - { url = "https://files.pythonhosted.org/packages/d8/48/a243d30223b024bc6057abe472b002cff01e97efefb4d2f0b0dcc5aece0b/duckdb-1.5.4-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02dd9f9a6124069213f13e3a474c208028c472fe1acdae12b38761f954fe4fc6", size = 19341849, upload-time = "2026-06-17T10:47:52.205Z" }, - { url = "https://files.pythonhosted.org/packages/08/ff/a5d48de4771e2403a8ef26a20dc7457b1c8f7e398ff0caf9c0cad8805f89/duckdb-1.5.4-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccc7f2694d02b4763fee61021d45e12f7bc5743993686563957df0cef799fbae", size = 21451698, upload-time = "2026-06-17T10:47:54.653Z" }, - { url = "https://files.pythonhosted.org/packages/79/b8/8244d7741b4afae67775cf0cb0d4eb9e923a83110907e4801e17fa078480/duckdb-1.5.4-cp311-cp311-win_amd64.whl", hash = "sha256:4c430e788d99b50854209bf2833ba36a45df75e57f86efb477046cd408bbd077", size = 13132643, upload-time = "2026-06-17T10:47:56.75Z" }, - { url = "https://files.pythonhosted.org/packages/e4/57/8169822a37f6dd7d561c567f9007e3cf04bf97bccb619afe90db849c0962/duckdb-1.5.4-cp311-cp311-win_arm64.whl", hash = "sha256:e2dc8340cfb6006025a798c50f40126d6e945a1d2487be94667bb4166556ce7b", size = 13986386, upload-time = "2026-06-17T10:47:59.345Z" }, - { url = "https://files.pythonhosted.org/packages/c8/f2/e2f4b477ae3a3b40e8b5f429832e48edb62ed9da99807cc4902e157e5646/duckdb-1.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:291a9e7502551170af989ff63139a7a49e99d68edbc5ef5017ac27541fe54c65", size = 32708876, upload-time = "2026-06-17T10:48:01.527Z" }, - { url = "https://files.pythonhosted.org/packages/2e/2b/b698d82a5e1e30b6a05748d72045f672994c6b22f4f0f8423523608b991f/duckdb-1.5.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:83e8c089bbb756ca4471d8b05943b80a106058697cf00615e70423106bb783bc", size = 17346125, upload-time = "2026-06-17T10:48:04.035Z" }, - { url = "https://files.pythonhosted.org/packages/71/75/37e13f39268eaf34864453b3a039c4a1ff0b088d3eae45a4289b41c98c1b/duckdb-1.5.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ff96d2a342b200e1ec6f1f19986c77f4ac16a49b6112f71c5b763989203a9d60", size = 15488133, upload-time = "2026-06-17T10:48:06.312Z" }, - { url = "https://files.pythonhosted.org/packages/cc/59/2d082af578f689231798245b54562c61416e49049b0bda81a06c56a4b53e/duckdb-1.5.4-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f935ef210ab00bc94bb1e3052697adaa36bb0ce7bdfeda8b0f34e2ff1643870", size = 19367895, upload-time = "2026-06-17T10:48:08.59Z" }, - { url = "https://files.pythonhosted.org/packages/52/2b/55c34d2863a76ca824ef8274691e84240b4ff1acde3d231709e82557c240/duckdb-1.5.4-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0cda263d8c20addb8d4f95464787cbe0af1144f7ab7e21db3709fb826ee01725", size = 21486499, upload-time = "2026-06-17T10:48:10.963Z" }, - { url = "https://files.pythonhosted.org/packages/cf/30/ade5952b8182fac86fab43b95ebe3836e66381d0ad64eb1e54bd8207c988/duckdb-1.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:266c7c909558ce7377f57d082cee408aadebdd9111be017558ca54e44a031037", size = 13147934, upload-time = "2026-06-17T10:48:13.061Z" }, - { url = "https://files.pythonhosted.org/packages/f5/00/278f0f70e25b9911afe2fd227b9460f2e6d76177f0dcc03f7f1454afefa5/duckdb-1.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:f14e79a006341f29ee5a2692a24dac5114e77533d579c57ec39124adf0135033", size = 13965235, upload-time = "2026-06-17T10:48:15.782Z" }, - { url = "https://files.pythonhosted.org/packages/da/69/3fcb34e523a9bad1f0557a6c7691a71ba66c43a05e5be9ee96a9a841ed65/duckdb-1.5.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:42a612e67d64450b446eb69695290d460713eef46e0f64467ab9dfe96264ee05", size = 32708366, upload-time = "2026-06-17T10:48:18.084Z" }, - { url = "https://files.pythonhosted.org/packages/f5/5f/bff5054c2c1d65decab36aa6296621e51a2a575a9f250db7ab9b83a325d6/duckdb-1.5.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3fb6f07d54ecf4d0d3c5179a2361fdddfafa14de4fc42696de4632479b703421", size = 17345735, upload-time = "2026-06-17T10:48:20.67Z" }, - { url = "https://files.pythonhosted.org/packages/93/12/d1b2b344e9699246aada6f9de5156e708fb476e2780e5bff9b5d95fe11d9/duckdb-1.5.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0f32ad7e0286c1c29ab6c73b29118c86101f8eee46aae54f54d0b50916f542f6", size = 15488568, upload-time = "2026-06-17T10:48:23.038Z" }, - { url = "https://files.pythonhosted.org/packages/c1/d1/ac56c6096e3e95da60b2c5dd5a0f0eb5540a80622e2e4f8faab893ec4e96/duckdb-1.5.4-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:698ec90bd5d5538bd5f6d212a4b61af443d240703cf45f134738535026556ea5", size = 19368184, upload-time = "2026-06-17T10:48:25.601Z" }, - { url = "https://files.pythonhosted.org/packages/1e/0b/2ae4c3e157a19d9b4ac1f09a5dea6f93012334cc2db09f1e0c71eb99693d/duckdb-1.5.4-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136cea7f886b78caf4035485b4b1e766e8b309e999f9e83a966f81ebb8122844", size = 21486523, upload-time = "2026-06-17T10:48:27.817Z" }, - { url = "https://files.pythonhosted.org/packages/64/7b/c3d8d21e0d0db8faa81eeeb3a55b9932f5a0a16466cb968dc713a653d701/duckdb-1.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:bd6777e8ddd74fb603a6d09766bfcff28638189f8aaa61fc0dffd9e9a4baa8e5", size = 13147807, upload-time = "2026-06-17T10:48:30.017Z" }, - { url = "https://files.pythonhosted.org/packages/44/48/ddf8d3740e3d28582944f70d84e720b5dc28c10ec22b668a0e0bd965f2f2/duckdb-1.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:73f4878a3012283024a64a1909e440aac12091ef336f671fc142f7e87449ce0c", size = 13965189, upload-time = "2026-06-17T10:48:32.251Z" }, - { url = "https://files.pythonhosted.org/packages/62/01/67ac4cbc8e552a1e14c029b5c443d828e68f94d5d913c574f577e1db277e/duckdb-1.5.4-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:4647968629d0677bbcc2416c7aeda8685eb84e4ca15a6dbd4f82a66cfc91a532", size = 32714364, upload-time = "2026-06-17T10:48:34.724Z" }, - { url = "https://files.pythonhosted.org/packages/4e/0e/eb44d983fa56b175f971eea251bde284a36d26cbb93fcb68035061f54078/duckdb-1.5.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e8fcef301cf68d3951ea1eb8ac4d76cea0a6f6a08f4c78fe4026fc96d217bebc", size = 17349820, upload-time = "2026-06-17T10:48:37.126Z" }, - { url = "https://files.pythonhosted.org/packages/10/b2/b9dc7624b105d414585b8530451c1162c0b4750c0be9be2e497bb47a8a9b/duckdb-1.5.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f6f39cd0dc6948dee17fd130aec55114f97a8ef6e1db519b9774087962bc5c8c", size = 15498160, upload-time = "2026-06-17T10:48:40.032Z" }, - { url = "https://files.pythonhosted.org/packages/b7/57/61356444f6a8c62dec3c3d129abfc53f428de1d484093d1bb381db441231/duckdb-1.5.4-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:262f068158beb5943f2c618f4e54b46db8306b959f90dce956f90a89f613673d", size = 19374183, upload-time = "2026-06-17T10:48:42.698Z" }, - { url = "https://files.pythonhosted.org/packages/b0/f4/d5d633dd7c5138d8f7c434e6ac2553c831b7fb658494efa8d0bc73df8623/duckdb-1.5.4-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d2307a76d199077b0055b354e90e857479461a0d875437535dd4833172c8b6d", size = 21487202, upload-time = "2026-06-17T10:48:45.407Z" }, - { url = "https://files.pythonhosted.org/packages/c0/26/5be13bbd5c3421dccfc1ad4ca9da4b97c5a3ddd73f66542092f3167ec52c/duckdb-1.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:6dcbb81a1276bc48deb4d562bce4f8895e4fc6348750a096e30052345c6d6552", size = 13666989, upload-time = "2026-06-17T10:48:47.764Z" }, - { url = "https://files.pythonhosted.org/packages/dc/82/4d52f3f9f9703a226b26b80bdae3f6905aeefe5221bf1815fc93ff02ca25/duckdb-1.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:0f8722346024e5d9f02b58bf7b0491a629f97fdc8a04a10e432940f471ee387a", size = 14449863, upload-time = "2026-06-17T10:48:50.18Z" }, +dependencies = [ + { name = "duckdb" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/10/a1ed10340ce58ee8c5c3bb7c351822c027c34bd6454fd4bff875ccf06231/duckdb_extension_lance-1.5.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:93b089f67105a82381e33ed1b5f1a81de0549b1b047eae2c7a16296a33aabe65", size = 57438352, upload-time = "2026-08-10T20:10:54.208Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c7/af4b76e2ef2bf33e73e51c86e1ca44b195e0c99324080f6e71b1e8dfc0f4/duckdb_extension_lance-1.5.5-py3-none-manylinux2014_aarch64.whl", hash = "sha256:ff08a1ad0530481c6eeac736c1c067c67afb47f0644ba973cf16fc2483199946", size = 77073030, upload-time = "2026-08-10T20:10:57.884Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5d/90da4d60f529c89311a747d6b3b32313f2c21e29dbe524378b6ace81d422/duckdb_extension_lance-1.5.5-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6da31e51a642b0ce6926a829be16810f03852558ee97cc5e3beee66749c98bee", size = 81091742, upload-time = "2026-08-10T20:11:01.534Z" }, + { url = "https://files.pythonhosted.org/packages/69/29/5b2c8b93a3889cdd533f79320cf3aa6f7ab1651f9e6d545dd70a7dd8725f/duckdb_extension_lance-1.5.5-py3-none-win_amd64.whl", hash = "sha256:2efe159ead14c4a9076657cd9c78dba7053d7f63766c4497e73e523e7b3311d4", size = 57947203, upload-time = "2026-08-10T20:11:04.871Z" }, ] [[package]] @@ -1031,6 +1045,7 @@ dependencies = [ { name = "datasets" }, { name = "docling-core" }, { name = "duckdb" }, + { name = "duckdb-extension-lance" }, { name = "elasticsearch8", extra = ["async"] }, { name = "fastapi" }, { name = "filelock" }, @@ -1115,6 +1130,7 @@ requires-dist = [ { name = "datasets", specifier = ">=3.6.0" }, { name = "docling-core", specifier = ">=2.91.0,<3.0.0" }, { name = "duckdb", specifier = ">=1.5.4" }, + { name = "duckdb-extension-lance", specifier = ">=1.5.5" }, { name = "elasticsearch8", extras = ["async"], specifier = ">=8.7.0" }, { name = "fastapi", specifier = ">=0.115.0" }, { name = "filelock", specifier = ">=3.25.2" }, From 5e99252119085b7453d418d4fa247dfb5065869b Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Tue, 8 Sep 2026 22:32:41 -0700 Subject: [PATCH 15/16] docs: update gotchas section for clarity on helper function guidelines --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 25a696993..d37962b99 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -64,4 +64,4 @@ tag atomically. The version lives in three files — always change it with See `docs/architecture/deployment.md` for the full pipeline ## Gotchas & Rules -- **Check the library before writing a helper.** Before adding any function that renders, serializes, parses or walks a `DoclingDocument`, or splits chunks or fuses scores, check `docling_core.transforms.serializer.*` / `DocItem.export_to_*`, `docling_core.transforms.chunker`, and the DuckDB `lance` extension first. Phase 1 of retrieval shipped a `` serializer, a caption geometry join and a label→markdown CASE that docling already did; all were deleted. Functions with one consumer get inlined. +- **Check the library before writing a helper.** Before adding any function that renders, serializes, parses or walks a `DoclingDocument`, or splits chunks or fuses scores. From ed31a8153336ac1b88dd8e9c97b8e5b3e3c2f708 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Tue, 8 Sep 2026 22:56:44 -0700 Subject: [PATCH 16/16] fix(ocr): file page headers and footers under the furniture content layer Docling's chunkers only skip items whose content_layer is FURNITURE; the PAGE_HEADER/PAGE_FOOTER label alone does nothing. Mirror what docling's own parsers do so running heads and page numbers never land in a chunk, while the layout API still returns them. --- .../src/extralit_server/contexts/ocr/text.py | 8 ++++++-- .../unit/contexts/ocr/test_docling_builder.py | 20 ++++++++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/extralit-server/src/extralit_server/contexts/ocr/text.py b/extralit-server/src/extralit_server/contexts/ocr/text.py index 96c239226..69c4f9845 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/text.py +++ b/extralit-server/src/extralit_server/contexts/ocr/text.py @@ -4,7 +4,7 @@ from typing import Optional -from docling_core.types.doc import DocItemLabel, DoclingDocument +from docling_core.types.doc import ContentLayer, DocItemLabel, DoclingDocument from docling_core.types.doc.document import NodeItem from extralit_server.contexts.ocr.docling_builder import ( @@ -15,6 +15,9 @@ make_prov, ) +# The labels docling itself files as furniture; a label alone does not keep them out of chunks. +FURNITURE_LABELS = frozenset({DocItemLabel.PAGE_HEADER, DocItemLabel.PAGE_FOOTER}) + def add_text_block( doc: DoclingDocument, @@ -36,4 +39,5 @@ def add_text_block( if block.label == DocItemLabel.SECTION_HEADER: return doc.add_heading(text=text, level=block.level or 1, prov=prov, parent=parent) - return doc.add_text(label=block.label, text=text, prov=prov, parent=parent) + layer = ContentLayer.FURNITURE if block.label in FURNITURE_LABELS else ContentLayer.BODY + return doc.add_text(label=block.label, text=text, prov=prov, parent=parent, content_layer=layer) diff --git a/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py b/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py index 22e9e1721..f079411fd 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py +++ b/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py @@ -1,7 +1,8 @@ """Tests for the DoclingDocument builder seam shared by every layout parser.""" import pytest -from docling_core.types.doc import BoundingBox, CoordOrigin, DocItemLabel, Size, TableCell +from docling_core.transforms.chunker import HierarchicalChunker +from docling_core.types.doc import BoundingBox, ContentLayer, CoordOrigin, DocItemLabel, Size, TableCell from extralit_server.contexts.ocr.docling_builder import ( LayoutBlock, @@ -318,6 +319,23 @@ def test_titles_become_title_items(self, doc, ctx): assert doc.texts[0].label == DocItemLabel.TITLE + def test_page_headers_and_footers_are_furniture(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.PAGE_HEADER, bbox=bbox(t=0, b=20), text="Running head"), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=40, b=60), text="body"), + LayoutBlock(label=DocItemLabel.PAGE_FOOTER, bbox=bbox(t=770, b=790), text="Page 1"), + ] + + append_blocks(doc, ctx, blocks) + + layers = {t.text: t.content_layer for t in doc.texts} + assert layers == { + "Running head": ContentLayer.FURNITURE, + "body": ContentLayer.BODY, + "Page 1": ContentLayer.FURNITURE, + } + assert [c.text for c in HierarchicalChunker().chunk(doc)] == ["body"] + def test_empty_text_blocks_are_skipped(self, doc, ctx): blocks = [ LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text=" "),