diff --git a/src/ndi/cloud/api/files.py b/src/ndi/cloud/api/files.py index 2a5c6fae..d77d4f4b 100644 --- a/src/ndi/cloud/api/files.py +++ b/src/ndi/cloud/api/files.py @@ -31,6 +31,23 @@ _TERMINAL_BULK_STATES = ("complete", "failed") _ACTIVE_BULK_STATES = ("queued", "extracting") +# Terminal job states reported by the file-tier service. 'superseded' is +# unique to file-tier: a later job for the same file has taken over, so +# polling on is pointless. See ndi-cloud-node/manuals/file-tier-design.md. +_TERMINAL_FILE_TIER_STATES = ("completed", "failed", "superseded") + +# Target tiers accepted by the server. PSEUDO_COLD is a non-production +# server-side test target that skips S3 entirely; it is not exposed here as +# a normal option but is accepted at the wire level for callers who need it +# in test environments. See ndi-cloud-node/manuals/file-tier-design.md. +FileTier = Literal[ + "STANDARD", + "STANDARD_IA", + "GLACIER_IR", + "GLACIER", + "DEEP_ARCHIVE", +] + @_auto_client @validate_call(config=VALIDATE_CONFIG) @@ -708,3 +725,244 @@ def waitForAllBulkUploads( } time.sleep(interval) interval = min(interval * backoff_factor, max_interval) + + +# --------------------------------------------------------------------------- +# File-tier family +# +# Kick off, poll, wait on, and read back the "move these documents' files to +# storage class X" workflow. See ndi-cloud-node/manuals/file-tier-design.md +# for the full design (warmest-wins shared-UID rule, fencing tokens, +# two-phase thaw semantics, terminal states). +# --------------------------------------------------------------------------- + + +@_auto_client +@validate_call(config=VALIDATE_CONFIG) +def setFileTier( + dataset_id: CloudId, + document_ids: list[str] | tuple[str, ...] | str, + target_tier: str, + *, + id_namespace: Literal["auto", "cloud", "ndi"] = "auto", + client: _Client = None, +) -> dict[str, Any]: + """POST /datasets/{datasetId}/file-tier-jobs + + Kick off an async job that moves every file referenced by *document_ids* + to *target_tier*. Documents may be addressed by cloud ``_id`` or NDI id; + the server infers each selector's namespace unless *id_namespace* forces + it. Returns the accepted-job envelope with ``jobId``, ``fileCount``, + ``resolvedDocumentCount``, and ``collateralDocumentIds``. See + ndi-cloud-node/manuals/file-tier-design.md. + + Args: + dataset_id: The cloud dataset id. + document_ids: One document id, a list/tuple of ids, or a mix of + cloud (24-hex) and NDI ids. Must be non-empty. + target_tier: Target S3 storage class. One of ``"STANDARD"``, + ``"STANDARD_IA"``, ``"GLACIER_IR"``, ``"GLACIER"``, + ``"DEEP_ARCHIVE"``. ``"PSEUDO_COLD"`` is accepted at the wire + level and used by the server-side non-production test path; + production stages reject it as ``INVALID_TARGET_TIER``. + id_namespace: ``"auto"`` (default) lets the server infer each id's + shape (24-hex → cloud, else NDI). Pass ``"cloud"`` or ``"ndi"`` + to force one interpretation for every selector. + client: Authenticated cloud client (auto-created if omitted). + + Returns: + Dict with ``jobId``, ``fileCount``, ``resolvedDocumentCount``, + ``collateralDocumentIds``. Pass ``jobId`` to + :func:`waitForFileTierJob` (or :func:`getFileTierJob` for one poll) + to follow the job to completion. + + MATLAB equivalent: +cloud/+api/+files/setFileTier.m + """ + if isinstance(document_ids, str): + ids: list[str] = [document_ids] + else: + ids = [str(d) for d in document_ids] + if not ids: + raise ValueError("document_ids must be non-empty") + + selectors: list[dict[str, str]] = [] + for did in ids: + sel: dict[str, str] = {"id": did} + if id_namespace != "auto": + sel["kind"] = id_namespace + selectors.append(sel) + + payload = { + "targetTier": target_tier, + "documentSelectors": selectors, + } + return client.post( + "/datasets/{datasetId}/file-tier-jobs", + json=payload, + datasetId=dataset_id, + ) + + +@_auto_client +@validate_call(config=VALIDATE_CONFIG) +def getFileTierJob( + job_id: NonEmptyStr, + *, + client: _Client = None, +) -> dict[str, Any]: + """GET /file-tier-jobs/{jobId} -- one poll of an async file-tier job. + + Returns a dict with fields ``jobId``, ``datasetId``, ``state``, + ``targetTier``, ``fileCount``, ``filesDone``, ``filesFailed``, + ``perPhaseCounts``, ``errors``, ``createdAt``, ``updatedAt``. Terminal + states are ``'completed'``, ``'failed'``, and ``'superseded'``. See + ndi-cloud-node/manuals/file-tier-design.md. + + MATLAB equivalent: +cloud/+api/+files/getFileTierJob.m + """ + return client.get("/file-tier-jobs/{jobId}", jobId=job_id) + + +@_auto_client +@validate_call(config=VALIDATE_CONFIG) +def waitForFileTierJob( + job_id: NonEmptyStr, + *, + timeout: float = 600.0, + initial_interval: float = 3.0, + max_interval: float = 30.0, + backoff_factor: float = 2.0, + client: _Client = None, +) -> dict[str, Any]: + """Poll a file-tier job until it finishes or times out. + + Repeatedly calls :func:`getFileTierJob` at exponentially growing + intervals until the job reaches a terminal state (``'completed'``, + ``'failed'``, or ``'superseded'``) or the overall timeout elapses. A + transient API failure is NOT treated as terminal -- a gateway blip + would otherwise be mistaken for a dead job. + + Args: + job_id: The file-tier job identifier from :func:`setFileTier`. + timeout: Overall deadline in seconds. Default 600 -- tier moves + fan out to many CopyObject calls; give them room. + initial_interval: First sleep between polls (s). Default 3. + max_interval: Cap on the per-poll sleep (s). Default 30. + backoff_factor: Multiplier applied after each poll. Default 2. + + Returns: + The last status dict from the server. On timeout, the returned + dict has ``state='timeout'`` and ``elapsed`` set to the + wall-clock seconds spent polling. The caller decides success by + reading ``state`` (``'completed'`` is the only "job did what you + asked" verdict; ``'failed'`` and ``'superseded'`` are terminal + non-successes). + + MATLAB equivalent: +cloud/+api/+files/waitForFileTierJob.m + """ + start = time.monotonic() + interval = initial_interval + last: Any = None + while True: + elapsed = time.monotonic() - start + try: + status = getFileTierJob(job_id, client=client) + last = status + state = status.get("state", "") if hasattr(status, "get") else "" + if state in _TERMINAL_FILE_TIER_STATES: + return status + except Exception: + # A failed poll is not a failed job. The gateway (Lambda 29 s + # cap) or an in-flight worker restart can drop one read; keep + # polling until the deadline rather than reporting the job + # dead on the first blip. + pass + if elapsed + interval > timeout: + payload: dict[str, Any] + if last is not None and hasattr(last, "data") and isinstance(last.data, dict): + payload = dict(last.data) + elif isinstance(last, dict): + payload = dict(last) + else: + payload = {} + payload["state"] = "timeout" + payload["elapsed"] = time.monotonic() - start + return payload + time.sleep(interval) + interval = min(interval * backoff_factor, max_interval) + + +@_auto_client +@validate_call(config=VALIDATE_CONFIG) +def getFileTier( + dataset_id: CloudId, + document_id: CloudId, + *, + client: _Client = None, +) -> dict[str, Any]: + """Read the cached files-tier summary for one document. + + Answers "what tier are this document's files on?" from the doc's + cached ``filesTier`` summary. Backed by + ``GET /datasets/{d}/documents/{doc}`` via + :func:`~ndi.cloud.api.documents.getDocument` -- there is no dedicated + tier-read endpoint because the server keeps a per-doc summary + alongside the doc, recomputed by the tier worker after every job. See + ndi-cloud-node/manuals/file-tier-design.md. + + Args: + dataset_id: The cloud dataset id. + document_id: The cloud API id of the document. + client: Authenticated cloud client (auto-created if omitted). + + Returns: + Dict with fields: + ``counts`` - per-tier file-count map (``STANDARD`` | + ``STANDARD_IA`` | ``GLACIER_IR`` | ``GLACIER`` | + ``DEEP_ARCHIVE`` | ``PSEUDO_COLD`` -> int). + Empty dict when the doc has never had a tier + operation. + ``dominant`` - the coldest tier with a non-zero count (the + design's ``dominantFileTier`` shorthand). ``""`` + if no tier state yet. + ``notes`` - list of divergence notes, e.g. "warmest-wins + overruled a freeze because sibling doc needs + file warm". Empty when clean. + ``updatedAt`` - timestamp of the last summary write, or + ``None`` until the first tier job runs. + ``raw`` - the full get-document response, for callers + that want the whole doc. + + MATLAB equivalent: +cloud/+api/+files/getFileTier.m + """ + from . import documents as docs_api + + doc = docs_api.getDocument(dataset_id, document_id, client=client) + + result: dict[str, Any] = { + "counts": {}, + "dominant": "", + "notes": [], + "updatedAt": None, + "raw": doc, + } + + summary = doc.get("filesTier") if hasattr(doc, "get") else None + if isinstance(summary, dict): + counts = summary.get("counts") + if isinstance(counts, dict) and counts: + result["counts"] = dict(counts) + dominant = summary.get("dominant") + if dominant: + result["dominant"] = str(dominant) + notes = summary.get("notes") + if notes: + if isinstance(notes, (list, tuple)): + result["notes"] = [str(n) for n in notes] + else: + result["notes"] = [str(notes)] + updated_at = summary.get("updatedAt") + if updated_at: + result["updatedAt"] = updated_at + + return result diff --git a/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml b/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml index 0c32886f..d637cb17 100644 --- a/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml +++ b/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml @@ -1072,6 +1072,129 @@ functions: Sync-pipeline callers use this to ensure remote inventories are stable before reading them. + # --- file-tier family (files.py) --- + - name: setFileTier + matlab_path: "+ndi/+cloud/+api/+files/setFileTier.m" + matlab_last_sync_hash: "6fdec7e7" + python_path: "ndi/cloud/api/files.py" + input_arguments: + - name: dataset_id + type_matlab: "string" + type_python: "CloudId" + - name: document_ids + type_matlab: "string | cellstr" + type_python: "list[str] | tuple[str, ...] | str" + - name: target_tier + type_matlab: "string" + type_python: "str" + - name: id_namespace + type_matlab: "string" + type_python: 'Literal["auto", "cloud", "ndi"]' + default: '"auto"' + - name: client + type_python: "_Client" + default: "None" + output_arguments: + - name: result + type_python: "dict[str, Any]" + decision_log: > + Ported from NDI-matlab#990 (matlab HEAD 6fdec7e7). POST + /datasets/{datasetId}/file-tier-jobs kicks off an async job that + moves the files referenced by a list of documents to a target S3 + storage class. Accepts a single id, a list, or a tuple; selectors + are built the same way MATLAB builds them (id + optional kind), + and idNamespace="auto" (default) suppresses the kind field so the + server can infer per-selector. target_tier is a plain str at the + wire level to admit PSEUDO_COLD in test environments -- the + docstring names the five production tiers explicitly. The + convenience FileTier Literal is exposed for callers who want + strict typing. + + - name: getFileTierJob + matlab_path: "+ndi/+cloud/+api/+files/getFileTierJob.m" + matlab_last_sync_hash: "6fdec7e7" + python_path: "ndi/cloud/api/files.py" + input_arguments: + - name: job_id + type_matlab: "string" + type_python: "NonEmptyStr" + - name: client + type_python: "_Client" + default: "None" + output_arguments: + - name: result + type_python: "dict[str, Any]" + decision_log: > + Ported from NDI-matlab#990 (matlab HEAD 6fdec7e7). GET + /file-tier-jobs/{jobId} -- one poll of a file-tier job. Terminal + states are 'completed', 'failed', 'superseded'; see + ndi-cloud-node/manuals/file-tier-design.md. + + - name: waitForFileTierJob + matlab_path: "+ndi/+cloud/+api/+files/waitForFileTierJob.m" + matlab_last_sync_hash: "6fdec7e7" + python_path: "ndi/cloud/api/files.py" + input_arguments: + - name: job_id + type_matlab: "string" + type_python: "NonEmptyStr" + - name: timeout + type_matlab: "double" + type_python: "float" + default: "600.0" + - name: initial_interval + type_matlab: "double" + type_python: "float" + default: "3.0" + - name: max_interval + type_matlab: "double" + type_python: "float" + default: "30.0" + - name: backoff_factor + type_matlab: "double" + type_python: "float" + default: "2.0" + - name: client + type_python: "_Client" + default: "None" + output_arguments: + - name: result + type_python: "dict[str, Any]" + decision_log: > + Ported from NDI-matlab#990 (matlab HEAD 6fdec7e7). Polls + getFileTierJob at exponentially growing intervals until the job + reaches a terminal state (completed | failed | superseded) or the + overall timeout elapses. Follows waitForBulkUpload's contract: + returns the last status dict with state='timeout' + elapsed on + timeout, letting the caller decide success by reading state. A + transient poll error (gateway blip) is NOT treated as terminal + -- the FileTierMockTest pin ports over as a mock unit test. + + - name: getFileTier + matlab_path: "+ndi/+cloud/+api/+files/getFileTier.m" + matlab_last_sync_hash: "6fdec7e7" + python_path: "ndi/cloud/api/files.py" + input_arguments: + - name: dataset_id + type_matlab: "string" + type_python: "CloudId" + - name: document_id + type_matlab: "string" + type_python: "CloudId" + - name: client + type_python: "_Client" + default: "None" + output_arguments: + - name: result + type_python: "dict[str, Any]" + decision_log: > + Ported from NDI-matlab#990 (matlab HEAD 6fdec7e7). Reads the + document's cached filesTier summary via documents.getDocument and + projects out {counts, dominant, notes, updatedAt, raw}. Absent + fields become empty defaults so callers can rely on the shape. + No dedicated tier-read endpoint exists; the server keeps a + per-doc summary alongside the doc. + # --- users.py --- - name: createUser matlab_path: "+ndi/+cloud/+api/+users/createUser.m" @@ -1363,7 +1486,7 @@ not_yet_ported: - name: url matlab_path: "+ndi/+cloud/+api/url.m" - matlab_last_sync_hash: "5804cd80d" + matlab_last_sync_hash: "6fdec7e7" status: ported_differently decision_log: > MATLAB URL builder. Python uses CloudClient._build_url() @@ -1381,6 +1504,11 @@ not_yet_ported: bulk-upload status in api/files.py, /advance is what compute.advanceSession already posts to, and the signed-URL-set gap is recorded on batchSignedUrlLookup and downloadGenericFiles. + 2026-09-16: bumped to 6fdec7e7 (NDI-matlab#990). Two rows added for the + file-tier routes (create_file_tier_job -> POST /datasets/{d}/file-tier-jobs, + get_file_tier_job -> GET /file-tier-jobs/{jobId}); both are inlined at the + call sites in api/files.py by setFileTier / getFileTierJob, so this entry + has nothing to add. - name: documentCount matlab_path: "+ndi/+cloud/+api/+documents/documentCount.m" diff --git a/tests/test_cloud_file_tier.py b/tests/test_cloud_file_tier.py new file mode 100644 index 00000000..74f6c9b4 --- /dev/null +++ b/tests/test_cloud_file_tier.py @@ -0,0 +1,415 @@ +"""Unit tests for ndi.cloud.api.files file-tier wrappers -- no network required. + +MATLAB counterpart: tests/+ndi/+unittest/FileTierMockTest.m (poll logic) plus +setFileTier / getFileTier request-shape assertions. + +The four wrappers under test (see files.py): + + setFileTier POST /datasets/{d}/file-tier-jobs + getFileTierJob GET /file-tier-jobs/{jobId} + waitForFileTierJob poll loop over getFileTierJob + getFileTier projects filesTier out of a GET /documents/{doc} + +The interesting behaviours here are ones a working server will not produce on +demand: a job that never reaches a terminal state, a timeout, a transient +API error a naive loop would mistake for a dead job, and each of the three +terminal states resolving with the right verdict. A live-cloud test can show +the happy path works; only mocks can show what happens when it does not. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + + +def _make_client() -> MagicMock: + """Return a mock CloudClient.""" + client = MagicMock() + client.config.org_id = "org-123" + client.config.api_url = "https://api.ndi-cloud.com/v1" + return client + + +# --------------------------------------------------------------------------- +# setFileTier -- request shape +# --------------------------------------------------------------------------- + + +class TestSetFileTier: + def test_posts_selector_list_to_file_tier_jobs(self): + from ndi.cloud.api.files import setFileTier + + client = _make_client() + client.post.return_value = { + "jobId": "j-1", + "fileCount": 2, + "resolvedDocumentCount": 1, + "collateralDocumentIds": [], + } + + result = setFileTier("ds-1", ["doc-a"], "GLACIER", client=client) + + client.post.assert_called_once() + call = client.post.call_args + assert call.args[0] == "/datasets/{datasetId}/file-tier-jobs" + assert call.kwargs["datasetId"] == "ds-1" + assert call.kwargs["json"] == { + "targetTier": "GLACIER", + "documentSelectors": [{"id": "doc-a"}], + } + assert result["jobId"] == "j-1" + + def test_accepts_a_bare_string_document_id(self): + from ndi.cloud.api.files import setFileTier + + client = _make_client() + client.post.return_value = {"jobId": "j-2"} + + setFileTier("ds-1", "doc-a", "STANDARD", client=client) + + payload = client.post.call_args.kwargs["json"] + assert payload["documentSelectors"] == [{"id": "doc-a"}] + + def test_id_namespace_cloud_forces_kind_on_every_selector(self): + from ndi.cloud.api.files import setFileTier + + client = _make_client() + client.post.return_value = {"jobId": "j-3"} + + setFileTier( + "ds-1", + ["doc-a", "doc-b"], + "DEEP_ARCHIVE", + id_namespace="cloud", + client=client, + ) + + payload = client.post.call_args.kwargs["json"] + assert payload["documentSelectors"] == [ + {"id": "doc-a", "kind": "cloud"}, + {"id": "doc-b", "kind": "cloud"}, + ] + + def test_id_namespace_ndi_forces_kind_on_every_selector(self): + from ndi.cloud.api.files import setFileTier + + client = _make_client() + client.post.return_value = {"jobId": "j-4"} + + setFileTier( + "ds-1", + ["ndi_abc"], + "GLACIER_IR", + id_namespace="ndi", + client=client, + ) + + payload = client.post.call_args.kwargs["json"] + assert payload["documentSelectors"] == [{"id": "ndi_abc", "kind": "ndi"}] + + def test_id_namespace_auto_never_writes_kind(self): + # The "auto" default is what makes mixed cloud/NDI id lists work -- + # a stray kind='auto' on the wire would be rejected by the server. + from ndi.cloud.api.files import setFileTier + + client = _make_client() + client.post.return_value = {"jobId": "j-5"} + + setFileTier("ds-1", ["doc-a"], "STANDARD_IA", client=client) + + payload = client.post.call_args.kwargs["json"] + assert "kind" not in payload["documentSelectors"][0] + + def test_empty_document_ids_raises(self): + from ndi.cloud.api.files import setFileTier + + with pytest.raises(ValueError, match="non-empty"): + setFileTier("ds-1", [], "STANDARD", client=_make_client()) + + def test_bad_id_namespace_raises(self): + # The Literal typing rejects anything but the three accepted values. + from ndi.cloud.api.files import setFileTier + + with pytest.raises(Exception): + setFileTier( + "ds-1", + ["doc-a"], + "STANDARD", + id_namespace="bogus", # type: ignore[arg-type] + client=_make_client(), + ) + + +# --------------------------------------------------------------------------- +# getFileTierJob -- URL shape +# --------------------------------------------------------------------------- + + +class TestGetFileTierJob: + def test_gets_file_tier_jobs_endpoint(self): + from ndi.cloud.api.files import getFileTierJob + + client = _make_client() + client.get.return_value = {"jobId": "j-1", "state": "running"} + + result = getFileTierJob("j-1", client=client) + + client.get.assert_called_once() + call = client.get.call_args + assert call.args[0] == "/file-tier-jobs/{jobId}" + assert call.kwargs["jobId"] == "j-1" + assert result["state"] == "running" + + +# --------------------------------------------------------------------------- +# waitForFileTierJob -- poll logic +# +# Mirrors FileTierMockTest.m. Uses monkeypatched getFileTierJob and a +# time.sleep stub so backoff is exercised without actually sleeping. +# --------------------------------------------------------------------------- + + +class _ScriptedGetFileTierJob: + """Return a sequence of scripted responses across successive calls. + + Each entry is either a dict (returned as the status payload) or an + Exception (raised, to model a transient poll failure). The final entry + repeats forever, so "always running" is one entry rather than a guess + about how many polls the wait will make. + """ + + def __init__(self, script): + self.script = list(script) + self.calls = 0 + + def __call__(self, job_id, *, client=None): + self.calls += 1 + item = self.script[min(self.calls - 1, len(self.script) - 1)] + if isinstance(item, Exception): + raise item + return item + + +@pytest.fixture() +def no_sleep(monkeypatch): + """Neutralise time.sleep inside ndi.cloud.api.files so tests run fast. + + monotonic still advances by wall clock, which is what the timeout + branch reads -- a fake clock would need a much heavier harness. + """ + monkeypatch.setattr("ndi.cloud.api.files.time.sleep", lambda _s: None) + + +class TestWaitForFileTierJob: + def test_completed_is_success(self, no_sleep): + from ndi.cloud.api import files as files_api + + scripted = _ScriptedGetFileTierJob( + [ + {"state": "completed", "fileCount": 3, "filesDone": 3}, + ] + ) + + with patch.object(files_api, "getFileTierJob", scripted): + result = files_api.waitForFileTierJob( + "j1", + timeout=5, + initial_interval=0.01, + max_interval=0.01, + client=_make_client(), + ) + + assert result["state"] == "completed" + assert scripted.calls == 1 + + def test_failed_is_terminal_not_retried(self, no_sleep): + from ndi.cloud.api import files as files_api + + scripted = _ScriptedGetFileTierJob([{"state": "failed"}]) + + with patch.object(files_api, "getFileTierJob", scripted): + result = files_api.waitForFileTierJob( + "j1", + timeout=5, + initial_interval=0.01, + max_interval=0.01, + client=_make_client(), + ) + + assert result["state"] == "failed" + assert scripted.calls == 1, "a failed job must not be polled again" + + def test_superseded_is_terminal_not_retried(self, no_sleep): + # 'superseded' is unique to file-tier: a later job for the same + # file has taken over. The caller has lost the race; polling on is + # pointless. + from ndi.cloud.api import files as files_api + + scripted = _ScriptedGetFileTierJob([{"state": "superseded"}]) + + with patch.object(files_api, "getFileTierJob", scripted): + result = files_api.waitForFileTierJob( + "j1", + timeout=5, + initial_interval=0.01, + max_interval=0.01, + client=_make_client(), + ) + + assert result["state"] == "superseded" + assert scripted.calls == 1, "a superseded job must not be polled again" + + def test_job_is_polled_until_it_completes(self, no_sleep): + from ndi.cloud.api import files as files_api + + scripted = _ScriptedGetFileTierJob( + [ + {"state": "queued"}, + {"state": "running"}, + {"state": "completed"}, + ] + ) + + with patch.object(files_api, "getFileTierJob", scripted): + result = files_api.waitForFileTierJob( + "j1", + timeout=5, + initial_interval=0.01, + max_interval=0.01, + client=_make_client(), + ) + + assert result["state"] == "completed" + assert scripted.calls == 3 + + def test_job_that_never_finishes_times_out(self, no_sleep): + from ndi.cloud.api import files as files_api + + scripted = _ScriptedGetFileTierJob([{"state": "running"}]) + + with patch.object(files_api, "getFileTierJob", scripted): + result = files_api.waitForFileTierJob( + "j1", + timeout=0.05, + initial_interval=0.01, + max_interval=0.01, + client=_make_client(), + ) + + assert result["state"] == "timeout" + assert "elapsed" in result + # We saw at least one running state before timing out; the last + # non-timeout fields ride through into the timeout payload. + assert result["elapsed"] > 0 + + def test_an_api_error_is_not_mistaken_for_a_terminal_state(self, no_sleep): + # A failed poll is not a failed job; keep polling until the deadline + # rather than reporting the job dead. A naive loop that returns on + # the first exception would break this. + from ndi.cloud.api import files as files_api + + scripted = _ScriptedGetFileTierJob([RuntimeError("gateway 504")]) + + with patch.object(files_api, "getFileTierJob", scripted): + result = files_api.waitForFileTierJob( + "j1", + timeout=0.5, + initial_interval=0.01, + max_interval=0.01, + client=_make_client(), + ) + + assert result["state"] == "timeout" + assert ( + scripted.calls > 1 + ), "a transient API error should be retried, not treated as terminal" + + +# --------------------------------------------------------------------------- +# getFileTier -- projects filesTier off a document GET +# --------------------------------------------------------------------------- + + +class TestGetFileTier: + def test_projects_files_tier_summary_out_of_document(self): + from ndi.cloud.api import files as files_api + + client = _make_client() + doc = { + "id": "doc-1", + "filesTier": { + "counts": {"GLACIER": 3, "STANDARD": 1}, + "dominant": "GLACIER", + "notes": ["sibling doc kept file warm"], + "updatedAt": "2026-09-16T12:00:00Z", + }, + } + with patch("ndi.cloud.api.documents.getDocument", return_value=doc) as mock_get: + result = files_api.getFileTier("ds-1", "doc-1", client=client) + + mock_get.assert_called_once_with("ds-1", "doc-1", client=client) + assert result["counts"] == {"GLACIER": 3, "STANDARD": 1} + assert result["dominant"] == "GLACIER" + assert result["notes"] == ["sibling doc kept file warm"] + assert result["updatedAt"] == "2026-09-16T12:00:00Z" + # raw carries the full document for callers that want it. + assert result["raw"] == doc + + def test_untiered_document_reads_clean_defaults(self): + # A doc that has never had a tier op should return an empty + # projection rather than raise -- callers can rely on the shape. + from ndi.cloud.api import files as files_api + + client = _make_client() + doc = {"id": "doc-1"} # no filesTier field at all + with patch("ndi.cloud.api.documents.getDocument", return_value=doc): + result = files_api.getFileTier("ds-1", "doc-1", client=client) + + assert result["counts"] == {} + assert result["dominant"] == "" + assert result["notes"] == [] + assert result["updatedAt"] is None + assert result["raw"] == doc + + def test_partial_files_tier_summary_projects_what_is_there(self): + # The server may write only `counts` and `dominant` on the first + # summary pass, before `notes` or `updatedAt` land. The projection + # must not confuse "missing" with "empty". + from ndi.cloud.api import files as files_api + + client = _make_client() + doc = { + "id": "doc-1", + "filesTier": { + "counts": {"STANDARD": 2}, + "dominant": "STANDARD", + }, + } + with patch("ndi.cloud.api.documents.getDocument", return_value=doc): + result = files_api.getFileTier("ds-1", "doc-1", client=client) + + assert result["counts"] == {"STANDARD": 2} + assert result["dominant"] == "STANDARD" + assert result["notes"] == [] + assert result["updatedAt"] is None + + def test_files_tier_notes_scalar_string_is_wrapped_in_a_list(self): + # The server encodes a single note as a string in some code paths. + # The projection normalises to a list so callers can always iterate. + from ndi.cloud.api import files as files_api + + client = _make_client() + doc = { + "id": "doc-1", + "filesTier": { + "counts": {"GLACIER": 1}, + "dominant": "GLACIER", + "notes": "just one note", + }, + } + with patch("ndi.cloud.api.documents.getDocument", return_value=doc): + result = files_api.getFileTier("ds-1", "doc-1", client=client) + + assert result["notes"] == ["just one note"]