From 2aa71e090af389a6a8f68d95f4df402624887f70 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 11:30:42 +0000 Subject: [PATCH] Enumerate a document's binaries the way NDI-matlab does file_uploads_for_document was reading files.file_info[].locations[]. location off the doc dict -- the authoring record, which DID's ingest invalidates. add_file_series writes the manifest to a tempfile and add_file with the default delete_original=True removes that tempfile once the bytes are stored under /. The upload then had nothing to send for the manifest, and every series' MEMBERS (which have no file_info entry at all) were silently missed -- run 227 on TEST_USER_1 caught the manifest half (Waltham-Data-Science/NDI-python#306); the member half was undetected until this reshape. NDI-matlab's +ndi/+database/+internal/list_binary_files.m has none of this trouble because it asks ndi.dataset.database_existbinarydoc for each name, which is DID's exist_doc -- the storage-side source of truth -- and iterates series_info(k).count from 1..count to generate member names (NAME_1, NAME_2, ...). Port the same shape: file_uploads_for_document takes an optional dataset kwarg; when given it walks file_info[i].name (yielding the manifest name for a series) plus series_info[k].count for members, and asks the dataset for each name's storage-side path. uid is the basename of that path -- DID keys files by uid at /, so fileparts is enough. Thread the dataset through uploadFilesForDatasetDocuments, through sync.operations._upload_binaries (all three callers: uploadNew, mirrorToRemote, twoWaySync), and through orchestration.uploadDataset. A caller that hands only a raw dict falls back to the recorded-location shape -- kept for isolated tests and for callers with no dataset. Retires PR #307's additional_roots patch, which was at the wrong layer and only recovered the manifest. Co-Authored-By: Claude Opus 4.7 Claude-Session: https://claude.ai/code/session_01AdGZ8fRPeY9JSpUHgXm9ao --- src/ndi/cloud/ndi_matlab_python_bridge.yaml | 18 + src/ndi/cloud/orchestration.py | 1 + src/ndi/cloud/sync/operations.py | 11 + src/ndi/cloud/upload.py | 152 ++++++-- ...est_cloud_upload_enumerates_via_dataset.py | 333 ++++++++++++++++++ 5 files changed, 491 insertions(+), 24 deletions(-) create mode 100644 tests/test_cloud_upload_enumerates_via_dataset.py diff --git a/src/ndi/cloud/ndi_matlab_python_bridge.yaml b/src/ndi/cloud/ndi_matlab_python_bridge.yaml index 5f163c16..49544692 100644 --- a/src/ndi/cloud/ndi_matlab_python_bridge.yaml +++ b/src/ndi/cloud/ndi_matlab_python_bridge.yaml @@ -921,6 +921,9 @@ functions: - name: documents type_matlab: "cell" type_python: "list[dict[str, Any]]" + - name: dataset + type_python: "Any | None" + default: "None" - name: client type_python: "CloudClient | None" default: "None" @@ -931,6 +934,21 @@ functions: MATLAB places this in +sync/+internal. Python places it in upload.py. Exact name match. + 2026-09-15: added a ``dataset`` kwarg. On MATLAB, +sync/+internal/ + uploadFilesForDatasetDocuments always has the dataset in hand -- it + calls +ndi/+database/+internal/list_binary_files, which uses + ndi_dataset.database_existbinarydoc (DID's exist_doc, the storage-side + source of truth) to locate each binary and enumerates series members + by iterating series_info(k).count. The Python counterpart was reading + the doc dict's authoring record (files.file_info[].locations[]. + location), which goes stale after DID ingest -- add_file with the + default delete_original=True removes the source and moves the bytes + to / -- and it never enumerated series members at all. + Both symptoms hit run 227 on TEST_USER_1 (Waltham-Data-Science/ + NDI-python#306). Passing the dataset in lets file_uploads_for_document + take the same MATLAB path when it is available; a caller without a + dataset falls back to the recorded-location shape (raw dict tests). + - name: uploadSingleFile matlab_path: "+ndi/+cloud/uploadSingleFile.m" matlab_last_sync_hash: "d9213ccb7" diff --git a/src/ndi/cloud/orchestration.py b/src/ndi/cloud/orchestration.py index 5d73f0a1..2e7e1310 100644 --- a/src/ndi/cloud/orchestration.py +++ b/src/ndi/cloud/orchestration.py @@ -539,6 +539,7 @@ def uploadDataset( client.config.org_id, cloud_id, doc_jsons, + dataset=dataset, client=client, ) if verbose: diff --git a/src/ndi/cloud/sync/operations.py b/src/ndi/cloud/sync/operations.py index 36c0ac77..7fcb4f25 100644 --- a/src/ndi/cloud/sync/operations.py +++ b/src/ndi/cloud/sync/operations.py @@ -332,6 +332,7 @@ def _upload_binaries( documents: list[dict[str, Any]], options: SyncOptions, *, + dataset: Any = None, client: CloudClient | None, ) -> set[str]: """Upload each document's binaries; return the ids whose binaries failed. @@ -341,6 +342,12 @@ def _upload_binaries( gets a 404 (NDI-matlab#805). uploadFilesForDatasetDocuments does not raise on a per-file failure -- it reports one -- so catching an exception was never going to see the ordinary case. + + ``dataset`` is passed through to ``uploadFilesForDatasetDocuments`` so + ``database_existbinarydoc`` can locate each binary at its storage-side + path (which is stable across ingest) and so series MEMBERS -- which have + no ``file_info`` entry -- are enumerated from ``series_info[k].count``. + See Waltham-Data-Science/NDI-python#306. """ if not options.sync_files or not documents: return set() @@ -351,6 +358,7 @@ def _upload_binaries( getattr(getattr(client, "config", None), "org_id", ""), cloud_dataset_id, documents, + dataset=dataset, client=client, ) except Exception as exc: # noqa: BLE001 @@ -621,6 +629,7 @@ def uploadNew( cloud_dataset_id, [documents[i] for i in uploaded if i in documents], options, + dataset=dataset, client=client, ) if binaries_failed: @@ -798,6 +807,7 @@ def mirrorToRemote( cloud_dataset_id, [documents[i] for i in uploaded if i in documents], options, + dataset=dataset, client=client, ) if binaries_failed: @@ -1062,6 +1072,7 @@ def twoWaySync( cloud_dataset_id, [documents[i] for i in uploaded if i in documents], options, + dataset=dataset, client=client, ) if binaries_failed: diff --git a/src/ndi/cloud/upload.py b/src/ndi/cloud/upload.py index 2d4e28ed..fad86a54 100644 --- a/src/ndi/cloud/upload.py +++ b/src/ndi/cloud/upload.py @@ -157,17 +157,21 @@ def uploadFilesForDatasetDocuments( dataset_id: str, documents: list[dict[str, Any]], *, + dataset: Any | None = None, client: CloudClient | None = None, ) -> dict[str, Any]: """Upload associated binary files for a list of documents. - For each document that has a ``file_uid`` field, obtains a - presigned URL and uploads the file. - Args: org_id: Organisation ID. dataset_id: Cloud dataset ID. documents: List of document property dicts. + dataset: The local ``ndi.dataset`` that produced *documents*. When + given, its ``database_existbinarydoc`` is the source of truth + for where each file actually lives on disk, and series members + (which have no ``file_info`` entry) are enumerated from + ``series_info[k].count``. Mirrors NDI-matlab's + ``list_binary_files.m``. client: Authenticated cloud client (auto-created if omitted). Returns: @@ -188,7 +192,7 @@ def uploadFilesForDatasetDocuments( for doc in documents: doc_id = document_id(doc) - for file_uid, file_path in file_uploads_for_document(doc): + for file_uid, file_path in file_uploads_for_document(doc, dataset=dataset): try: url = files_api.getFileUploadURL(org_id, dataset_id, file_uid, client=client) files_api.putFiles(url, file_path) @@ -202,34 +206,134 @@ def uploadFilesForDatasetDocuments( return report -def file_uploads_for_document(doc: dict[str, Any]) -> list[tuple[str, str]]: +def file_uploads_for_document( + doc: dict[str, Any], + *, + dataset: Any | None = None, +) -> list[tuple[str, str]]: """The ``(file_uid, local_path)`` pairs to upload for one document. - TWO SHAPES REACH THIS, AND ONLY ONE USED TO BE READ. - - A manifest entry carries ``file_uid`` and ``file_path`` at the top level, - and that is all this looked at. A real ``ndi.document`` carries neither: - its binaries live under ``files.file_info[].locations[]``, each location - holding a ``uid`` and a ``location`` that is a filesystem path until - :mod:`ndi.cloud.filehandler` rewrites it to ``ndic://``. So every caller - passing real documents -- which is every caller that enumerates a dataset - -- matched nothing and uploaded nothing, silently, because a document - with no recognised file is indistinguishable from a document with no - files at all. - - Locations already rewritten to ``ndic://`` are skipped: that scheme means - the file is on the cloud, which is the opposite of something to upload. - A location that no longer exists on disk is skipped too -- there is - nothing to send -- and the caller learns of it as a document whose - binaries did not arrive. + THREE SHAPES REACH THIS. + + * A manifest entry carrying top-level ``file_uid`` and ``file_path`` is + taken as-is: legacy input from callers that already know the pair. + + * A real ``ndi.document`` alongside its owning ``dataset``: the storage- + side path is asked for by name, via + :meth:`ndi.dataset.database_existbinarydoc` (which is DID's + ``exist_doc``). This mirrors NDI-matlab's + ``+ndi/+database/+internal/list_binary_files.m`` and is what the live + upload path always uses now. The advantage: it covers both the + recorded-location-is-gone-after-ingest case (Waltham-Data-Science/ + NDI-python#306 -- ``did.document.Document.add_file_series`` writes the + manifest to a tempfile, and ``add_file`` with the default + ``delete_original=True`` removes it once ingested) AND series MEMBERS, + which have no ``file_info`` entry and are enumerated from + ``series_info[k].count`` (1..count, member names ``NAME_i``). + + * A raw document dict WITHOUT a dataset: the fallback reads + ``files.file_info[].locations[].location`` and calls ``os.path.exists``. + Kept so isolated tests and callers with no dataset in hand behave the + way they used to. Locations already rewritten to ``ndic://`` are + skipped there: that scheme means the file is on the cloud, which is + the opposite of something to upload. """ - import os - top_uid = str(doc.get("file_uid", "") or "") top_path = str(doc.get("file_path", "") or "") if top_uid and top_path: return [(top_uid, top_path)] + if dataset is not None and callable(getattr(dataset, "database_existbinarydoc", None)): + return _file_uploads_via_dataset(doc, dataset) + + return _file_uploads_via_recorded_location(doc) + + +def _file_uploads_via_dataset( + doc: dict[str, Any], + dataset: Any, +) -> list[tuple[str, str]]: + """Enumerate binaries the way NDI-matlab's ``list_binary_files.m`` does. + + Ordinary files: walk ``files.file_info[i].name`` (which is what + ``current_file_list()`` returns). For a series document, this yields the + manifest name, and the manifest bytes live at ``/`` + after ingest -- exactly where ``database_existbinarydoc`` looks. + + Series members: iterate ``files.series_info[k].count`` from 1..count and + ask for ``NAME_i``. Slots are one-based (see DID's series API), an absent + slot is normal (a sparse series is the case the mechanism exists for), + and the manifest is what makes each slot's uid resolvable. + + ``uid = basename(path)``: DID's file store keys files by uid, and + ``exist_doc`` returns a path under ``/`` whose basename IS the + uid. NDI-matlab's ``list_binary_files.m`` does the same + (``fileparts(full_file_path)``). + """ + import os + + doc_id = document_id(doc) + if not doc_id: + return [] + files = doc.get("files") or {} + if not isinstance(files, dict): + return [] + + pairs: list[tuple[str, str]] = [] + + file_info = files.get("file_info") or [] + if isinstance(file_info, dict): + file_info = [file_info] + for entry in file_info: + if not isinstance(entry, dict): + continue + name = str(entry.get("name", "") or "") + if not name: + continue + found, path = dataset.database_existbinarydoc(doc_id, name) + if not found or path is None: + continue + path_str = str(path) + uid = os.path.basename(path_str) + if uid: + pairs.append((uid, path_str)) + + series_info = files.get("series_info") or [] + if isinstance(series_info, dict): + series_info = [series_info] + for entry in series_info: + if not isinstance(entry, dict): + continue + series_name = str(entry.get("name", "") or "") + if not series_name: + continue + try: + count = int(entry.get("count", 0) or 0) + except (TypeError, ValueError): + continue + for i in range(1, count + 1): + member_name = f"{series_name}_{i}" + found, path = dataset.database_existbinarydoc(doc_id, member_name) + if not found or path is None: + continue + path_str = str(path) + uid = os.path.basename(path_str) + if uid: + pairs.append((uid, path_str)) + + return pairs + + +def _file_uploads_via_recorded_location(doc: dict[str, Any]) -> list[tuple[str, str]]: + """The pre-#306 shape: read the authoring record off the doc dict. + + Kept for callers that pass a raw dict and no dataset. Never covers series + members (they have no ``file_info`` entry), and skips a location whose + file has been removed from disk -- both facts are what made the + dataset-driven path necessary in the first place. + """ + import os + pairs: list[tuple[str, str]] = [] files = doc.get("files") or {} if not isinstance(files, dict): diff --git a/tests/test_cloud_upload_enumerates_via_dataset.py b/tests/test_cloud_upload_enumerates_via_dataset.py new file mode 100644 index 00000000..84ab507b --- /dev/null +++ b/tests/test_cloud_upload_enumerates_via_dataset.py @@ -0,0 +1,333 @@ +"""``file_uploads_for_document`` asks the dataset where its binaries live. + +TWO CONCERNS, one MATLAB counterpart. + +1. A series manifest is written by ``did.document.Document.add_file_series`` + to ``tempfile.NamedTemporaryFile(prefix="did_manifest_")``, and + ``add_file`` with the default ``delete_original=True`` removes that + source once DID has ingested it. The doc's authoring record + (``files.file_info[].locations[].location``) still names the tempfile, + so a caller reading only that record uploads nothing for the manifest. + Run 227 on TEST_USER_1 caught it (Waltham-Data-Science/NDI-python#306). + +2. A series' MEMBERS have no ``file_info`` entry at all. They are named + ``NAME_i`` for ``i in 1..series_info[k].count`` and their bytes live at + ``/``. A caller that walks only ``file_info`` never + sees any of them, so an upload that took the recorded-location path + sent zero member files whichever way the manifest went. + +NDI-matlab's ``+ndi/+database/+internal/list_binary_files.m`` handles +both by calling ``dataset.database_existbinarydoc(doc_id, filename)`` for +each name -- that is DID's storage-side ``exist_doc``, which knows where +the bytes actually live -- and by iterating ``series_info(k).count`` to +generate member names. This test pins the Python counterpart: +``file_uploads_for_document(doc, dataset=)`` produces the +same pairs on the same shapes. + +Live-cloud coverage of the whole round trip is +``tests/test_cloud_file_series_round_trip.py``; this file is the offline +half, no credentials required. +""" + +from __future__ import annotations + +import os +import uuid + +from did.document import Document as DIDDocument + +from ndi.cloud.upload import file_uploads_for_document +from ndi.dataset import ndi_dataset_dir +from ndi.document import ndi_document + +SERIES_NAME = "chunkdata.bin" +MEMBER_COUNT = 4 + + +class _DatasetStub: + """A minimal stand-in for :class:`ndi.dataset` -- one method, a routing + table from ``(doc_id, filename)`` to ``(exists, path_or_None)``. + + The tests that use it don't need DID; they need to prove that the + enumerator asks ``database_existbinarydoc`` for the right names in the + right order, and passes its answers through. A real dataset covers the + end-to-end shape below. + """ + + def __init__(self, table): + self._table = dict(table) + self.calls: list[tuple[str, str]] = [] + + def database_existbinarydoc(self, doc_id, filename): + self.calls.append((doc_id, filename)) + return self._table.get((doc_id, filename), (False, None)) + + +def _series_doc_bare(doc_id, series_name, count): + """A doc dict shaped like an ingested series -- names only, no live paths. + + ``file_info`` names the manifest. ``series_info`` records ``count``. + Nothing here says where any of it lives; that is the dataset's job. + """ + return { + "base": {"id": doc_id}, + "files": { + "file_info": [ + { + "name": series_name, + "locations": [{"uid": "manifest-uid", "location": "/gone/manifest"}], + } + ], + "series_info": [ + {"name": series_name, "count": count}, + ], + }, + } + + +class TestTheDatasetIsAsked: + """The point of the reshape: ``database_existbinarydoc`` is the source + of truth. Ordinary files come from ``file_info[].name``; series members + come from ``series_info[k].count``, one-based.""" + + def test_the_manifest_name_is_looked_up_by_database_existbinarydoc(self, tmp_path): + doc = _series_doc_bare("doc1", SERIES_NAME, count=0) + cached = tmp_path / "manifest-uid" + cached.write_bytes(b"manifest bytes") + stub = _DatasetStub({("doc1", SERIES_NAME): (True, str(cached))}) + + pairs = file_uploads_for_document(doc, dataset=stub) + + assert pairs == [("manifest-uid", str(cached))] + assert stub.calls == [("doc1", SERIES_NAME)] + + def test_series_members_are_enumerated_from_series_info_count(self, tmp_path): + doc = _series_doc_bare("doc1", SERIES_NAME, count=MEMBER_COUNT) + # Manifest present, plus one file per member slot. + table = {("doc1", SERIES_NAME): (True, str(tmp_path / "manifest-uid"))} + (tmp_path / "manifest-uid").write_bytes(b"m") + for i in range(1, MEMBER_COUNT + 1): + uid = f"member-{i}-uid" + (tmp_path / uid).write_bytes(b"m") + table[("doc1", f"{SERIES_NAME}_{i}")] = (True, str(tmp_path / uid)) + stub = _DatasetStub(table) + + pairs = file_uploads_for_document(doc, dataset=stub) + + # Manifest first (from file_info), then members in slot order. + assert pairs == [ + ("manifest-uid", str(tmp_path / "manifest-uid")), + ("member-1-uid", str(tmp_path / "member-1-uid")), + ("member-2-uid", str(tmp_path / "member-2-uid")), + ("member-3-uid", str(tmp_path / "member-3-uid")), + ("member-4-uid", str(tmp_path / "member-4-uid")), + ] + # Slot names are one-based, as in DID's series API. + assert stub.calls == [ + ("doc1", SERIES_NAME), + ("doc1", f"{SERIES_NAME}_1"), + ("doc1", f"{SERIES_NAME}_2"), + ("doc1", f"{SERIES_NAME}_3"), + ("doc1", f"{SERIES_NAME}_4"), + ] + + def test_a_sparse_series_skips_absent_slots(self, tmp_path): + """A sparse series is the case the mechanism exists for (gene + pyramids write tiles 1, 4, 9, ...). An absent slot is normal; + producing no pair for it is the right answer, not an error.""" + doc = _series_doc_bare("doc1", SERIES_NAME, count=3) + (tmp_path / "manifest-uid").write_bytes(b"m") + (tmp_path / "member-2-uid").write_bytes(b"m") + stub = _DatasetStub( + { + ("doc1", SERIES_NAME): (True, str(tmp_path / "manifest-uid")), + # slot 1: absent + ("doc1", f"{SERIES_NAME}_2"): (True, str(tmp_path / "member-2-uid")), + # slot 3: absent + } + ) + + pairs = file_uploads_for_document(doc, dataset=stub) + + assert pairs == [ + ("manifest-uid", str(tmp_path / "manifest-uid")), + ("member-2-uid", str(tmp_path / "member-2-uid")), + ] + + def test_uid_is_the_basename_of_the_stored_path(self, tmp_path): + """DID's file store keys files by uid at ``/``. + ``exist_doc`` returns that path; the uid is its basename. + NDI-matlab's ``list_binary_files.m`` does the same + (``fileparts(full_file_path)``).""" + doc = { + "base": {"id": "doc1"}, + "files": {"file_info": [{"name": "plain.bin", "locations": []}]}, + } + stored = tmp_path / "abc123def" + stored.write_bytes(b"x") + stub = _DatasetStub({("doc1", "plain.bin"): (True, str(stored))}) + + pairs = file_uploads_for_document(doc, dataset=stub) + + assert pairs == [("abc123def", str(stored))] + + def test_matlab_shaped_single_element_dicts_still_work(self, tmp_path): + """MATLAB's ``jsonencode`` writes a one-element struct array as a + bare object. The enumerator has to normalise both ``file_info`` and + ``series_info`` for that case, or a series-of-one gets skipped.""" + doc = { + "base": {"id": "doc1"}, + "files": { + # Single-element -> bare object, not a list. + "file_info": { + "name": SERIES_NAME, + "locations": {"uid": "manifest-uid", "location": "/gone"}, + }, + "series_info": {"name": SERIES_NAME, "count": 1}, + }, + } + (tmp_path / "manifest-uid").write_bytes(b"m") + (tmp_path / "member-1-uid").write_bytes(b"m") + stub = _DatasetStub( + { + ("doc1", SERIES_NAME): (True, str(tmp_path / "manifest-uid")), + ("doc1", f"{SERIES_NAME}_1"): (True, str(tmp_path / "member-1-uid")), + } + ) + + pairs = file_uploads_for_document(doc, dataset=stub) + + assert pairs == [ + ("manifest-uid", str(tmp_path / "manifest-uid")), + ("member-1-uid", str(tmp_path / "member-1-uid")), + ] + + def test_a_doc_with_no_id_yields_nothing(self): + """``database_existbinarydoc`` needs an id; without one the + enumerator declines rather than probing for the empty string.""" + doc = _series_doc_bare("", SERIES_NAME, count=1) + stub = _DatasetStub({}) + + assert file_uploads_for_document(doc, dataset=stub) == [] + assert stub.calls == [] + + +class TestTheRawDictFallback: + """A raw dict without a dataset keeps the pre-#306 shape: read the + authoring record and skip anything gone. It never covers series members + (they have no ``file_info`` entry) -- but nothing was covering them + before either, and the dataset path is what does now.""" + + def test_a_live_recorded_location_is_returned(self, tmp_path): + live = tmp_path / "live.bin" + live.write_bytes(b"x") + doc = { + "files": { + "file_info": [ + {"name": "live.bin", "locations": [{"uid": "u", "location": str(live)}]} + ] + } + } + assert file_uploads_for_document(doc) == [("u", str(live))] + + def test_a_gone_location_is_skipped(self, tmp_path): + doc = { + "files": { + "file_info": [ + { + "name": "gone.bin", + "locations": [{"uid": "u", "location": str(tmp_path / "gone")}], + } + ] + } + } + assert file_uploads_for_document(doc) == [] + + def test_the_manifest_top_level_shape_still_works(self, tmp_path): + path = tmp_path / "m.bin" + path.write_bytes(b"x") + doc = {"file_uid": "u", "file_path": str(path)} + assert file_uploads_for_document(doc) == [("u", str(path))] + + def test_a_stub_without_database_existbinarydoc_falls_back(self, tmp_path): + """A caller that hands an object which is not a real dataset should + get the raw-dict behaviour, not a raise. Guards against a caller + forgetting to pass the real dataset and getting silence instead of + the fallback.""" + live = tmp_path / "live.bin" + live.write_bytes(b"x") + doc = { + "files": { + "file_info": [ + {"name": "live.bin", "locations": [{"uid": "u", "location": str(live)}]} + ] + } + } + assert file_uploads_for_document(doc, dataset=object()) == [("u", str(live))] + + +class TestEndToEndAgainstARealDataset: + """The actual failure mode #306 names, driven through the shape a live + upload takes: build a series doc, ingest it into a real dataset, and + ask the enumerator for its pairs. Both the manifest AND every member + must appear, keyed by the uids DID assigned them.""" + + def test_a_series_doc_yields_manifest_and_all_members(self, tmp_path): + local = ndi_dataset_dir("test_series_ds", str(tmp_path)) + members_dir = tmp_path / "members" + members_dir.mkdir() + member_paths = [] + for i in range(MEMBER_COUNT): + p = members_dir / f"m_{i + 1}.bin" + p.write_bytes(bytes((k * (i + 1)) % 251 for k in range(1, 65))) + member_paths.append(str(p)) + + seed = ndi_document( + "demoNDISeries", + **{ + "base.name": f"series_doc_{uuid.uuid4().hex[:8]}", + "demoNDISeries.value": 1, + }, + ) + did_doc = DIDDocument(seed.document_properties) + did_doc.add_file_series(SERIES_NAME, member_paths) + doc = ndi_document(did_doc.document_properties) + + local.database_add(doc) + + # The recorded manifest location must be gone -- that IS the shape + # this test exists to prove the enumerator survives. Read it back + # from the doc as stored and check. + stored = local.database_search(None) + stored_doc = next(d for d in stored if d.id == doc.id) + fi = stored_doc.document_properties["files"]["file_info"] + if isinstance(fi, dict): + fi = [fi] + manifest_entry = next(e for e in fi if e.get("name") == SERIES_NAME) + locs = manifest_entry.get("locations") + if isinstance(locs, dict): + locs = [locs] + recorded = locs[0].get("location", "") + assert not os.path.exists(recorded), ( + "fixture: DID ingest should have removed the manifest tempfile; " + "if this fires, the failure mode this test targets no longer " + "reproduces and the test is testing something else." + ) + + pairs = file_uploads_for_document(stored_doc.document_properties, dataset=local) + + # One pair for the manifest, one per member. Every uid resolves to a + # file under DID's FileDir -- so a live + # ``uploadFilesForDatasetDocuments`` call would send the manifest + # bytes and every member's bytes, keyed by their uids, and the + # download side finds each one where it looks. + assert ( + len(pairs) == MEMBER_COUNT + 1 + ), f"expected manifest + {MEMBER_COUNT} members; got {pairs!r}" + for uid, path in pairs: + assert uid, f"a pair with no uid: ({uid!r}, {path!r})" + assert os.path.isfile( + path + ), f"pair points at a path that does not exist: ({uid!r}, {path!r})" + # uid is the basename of the storage-side path -- DID keys by uid. + assert os.path.basename(path) == uid