From dbdedf68f8031ae7407ce8875342b74bf63f2095 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 23:29:06 +0000 Subject: [PATCH 1/3] Add file CRC-32 support for NDI Cloud: lookupFiles, checksummed uploads (ndi-cloud-node#162) Python half of VH-Lab/NDI-matlab#1025. - ndi.fun.file.crc32 / crc32ToBase64: a file's CRC-32 in the cloud's encoding (base64 of the 4-byte big-endian value), via zlib in chunks. - getFileUploadURL(..., checksum_crc32=) sends checksumCRC32; putFiles / putFileBytes(..., checksum_crc32=) send the x-amz-checksum-crc32 header that such a URL signs. Malformed values are refused. - ndi.cloud.api.files.lookupFiles: POST /files/lookup, split at 5000 uids. - uploadSingleFile sends the checksum by default (checksum=True). - getSignedURLSetAll merges the pages' fileInfo. - Bridge: lookupFiles, crc32, crc32ToBase64 recorded with matlab_branch claude/file-crc32-lookup; getFileUploadURL, putFiles, uploadSingleFile, listFiles(All), getSignedURLSet(All) and url synced to NDI-matlab 8fa6ea5e. - Tests: offline tests/test_cloud_api_files_crc32.py; live TestFileChecksums in tests/test_cloud_live.py (skips where the server predates the route). Two test_phase2_gaps uploadSingleFile tests pass checksum=False, since their file does not exist and the default now reads it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SwsqFA9Zs1HGm6YM4g2XRV --- CHANGELOG.md | 15 + src/ndi/cloud/api/files.py | 133 ++++++++- .../cloud/api/ndi_matlab_python_bridge.yaml | 67 ++++- src/ndi/cloud/ndi_matlab_python_bridge.yaml | 10 +- src/ndi/cloud/upload.py | 14 +- src/ndi/fun/file.py | 59 +++- src/ndi/fun/ndi_matlab_python_bridge.yaml | 40 +++ tests/test_cloud_api_files_crc32.py | 272 ++++++++++++++++++ tests/test_cloud_live.py | 139 +++++++++ tests/test_phase2_gaps.py | 9 +- 10 files changed, 739 insertions(+), 19 deletions(-) create mode 100644 tests/test_cloud_api_files_crc32.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f734caf..5c1af198 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Added + +- **File CRC-32 checksums for NDI Cloud** (ndi-cloud-node#162, mirroring + NDI-matlab#1025), so an importer can resume an interrupted upload by + re-uploading only the files whose cloud copy does not match. + `ndi.fun.file.crc32(path)` gives a file's CRC-32 in the cloud's encoding + (base64 of the 4-byte big-endian value). `getFileUploadURL` and `putFiles` + / `putFileBytes` take `checksum_crc32=`, so S3 rejects a corrupted upload + and the cloud records the value; `ndi.cloud.upload.uploadSingleFile` sends + it by default (`checksum=False` turns it off). New + `ndi.cloud.api.files.lookupFiles(dataset_id, uids)` returns `uploaded`, + `size` and `crc32` for given uids plus the `missing` ones, splitting lists + over the server's 5000-uid limit. `getSignedURLSetAll` merges the pages' + `fileInfo`. `listFiles` and `getFileDetails` already pass `crc32` through. + ### Fixed - **`ndi.probe.timeseries.readtimeseries()` names its first argument diff --git a/src/ndi/cloud/api/files.py b/src/ndi/cloud/api/files.py index 2e628dab..669e86ef 100644 --- a/src/ndi/cloud/api/files.py +++ b/src/ndi/cloud/api/files.py @@ -10,6 +10,7 @@ from __future__ import annotations +import re import time from pathlib import Path from typing import Annotated, Any, Literal @@ -27,6 +28,33 @@ _Client = Annotated[CloudClient | None, SkipValidation()] +# A CRC-32 as NDI Cloud and S3 encode it: base64 of the 4-byte big-endian +# value, so six base64 characters and '=='. See ndi.fun.file.crc32. +_CRC32_BASE64 = re.compile(r"^[A-Za-z0-9+/]{6}==$") + +# POST /files/lookup accepts at most this many uids per call; lookupFiles +# splits longer lists. +LOOKUP_MAX_UIDS = 5000 + + +def _checked_crc32(checksum_crc32: str) -> str: + if checksum_crc32 and not _CRC32_BASE64.match(checksum_crc32): + raise ValueError( + "checksum_crc32 must be the base64 of a 4-byte CRC-32 (e.g. 'NhCmhg=='), " + f"got {checksum_crc32!r}" + ) + return checksum_crc32 + + +def _upload_headers(checksum_crc32: str) -> dict[str, str]: + headers = {"Content-Type": "application/octet-stream"} + if checksum_crc32: + # The upload URL signs this header when it was requested with a + # checksum (NDI Cloud #162), so it must be sent, with the same value. + headers["x-amz-checksum-crc32"] = checksum_crc32 + return headers + + # Terminal job states reported by the bulk-upload service. _TERMINAL_BULK_STATES = ("complete", "failed") _ACTIVE_BULK_STATES = ("queued", "extracting") @@ -56,14 +84,31 @@ def getFileUploadURL( dataset_id: CloudId, file_uid: NonEmptyStr, *, + checksum_crc32: str = "", client: _Client = None, ) -> str: """GET /datasets/{organizationId}/{datasetId}/files/{file_uid} Returns a presigned S3 URL for uploading. + + Args: + org_id: The organization id. + dataset_id: The cloud dataset id. + file_uid: The uid the file will have. + checksum_crc32: Optional. The file's CRC-32 as base64 of its 4-byte + big-endian value (:func:`ndi.fun.file.crc32`). When given, the URL + signs the ``x-amz-checksum-crc32`` header, so the upload must send + it: pass the same value to :func:`putFiles`. S3 then rejects + content that does not match, and the cloud records the value as + the file's ``crc32``. + client: Authenticated cloud client (auto-created if omitted). + + MATLAB equivalent: +cloud/+api/+files/getFileUploadURL.m """ + params = {"checksumCRC32": _checked_crc32(checksum_crc32)} if checksum_crc32 else None result = client.get( "/datasets/{organizationId}/{datasetId}/files/{file_uid}", + params=params, organizationId=org_id, datasetId=dataset_id, file_uid=file_uid, @@ -100,6 +145,7 @@ def putFiles( job_id: str = "", wait_for_completion: bool = False, completion_timeout: float = 60.0, + checksum_crc32: str = "", ) -> bool: """PUT a local file to a presigned S3 URL. @@ -118,6 +164,12 @@ def putFiles( the signed PUT returning 200 already means done. completion_timeout: Overall wait-for-completion deadline, in seconds. Default 60. + checksum_crc32: The file's CRC-32 (:func:`ndi.fun.file.crc32`). + Required when the URL came from + ``getFileUploadURL(..., checksum_crc32=crc)``: that URL signs the + ``x-amz-checksum-crc32`` header, so it is sent with the same + value, and S3 rejects content that does not match. Default ``""`` + (no header). Returns: True on success (and, when ``wait_for_completion`` is True, the @@ -133,12 +185,13 @@ def putFiles( from ..exceptions import CloudUploadError assert_safe_transfer_url(url, what="upload URL") + headers = _upload_headers(_checked_crc32(checksum_crc32)) file_path = Path(file_path) with open(file_path, "rb") as fh: resp = requests.put( url, data=fh, - headers={"Content-Type": "application/octet-stream"}, + headers=headers, timeout=timeout, ) @@ -161,6 +214,8 @@ def putFileBytes( url: NonEmptyStr, data: bytes, timeout: int = 120, + *, + checksum_crc32: str = "", ) -> bool: """PUT raw bytes to a presigned S3 URL. @@ -168,6 +223,8 @@ def putFileBytes( url: Presigned URL. data: Bytes to upload. timeout: Request timeout in seconds. + checksum_crc32: The bytes' CRC-32 (:func:`ndi.fun.file.crc32ToBase64` + of ``zlib.crc32(data)``); see :func:`putFiles`. Returns: True on success. @@ -183,7 +240,7 @@ def putFileBytes( resp = requests.put( url, data=data, - headers={"Content-Type": "application/octet-stream"}, + headers=_upload_headers(_checked_crc32(checksum_crc32)), timeout=timeout, ) @@ -343,7 +400,8 @@ def listFiles( The page envelope with fields ``datasetId``, ``limit``, ``count``, ``cursor``, ``hasMore``, ``totalNumber``, and ``files`` -- a list of file summaries, each with ``uid``, ``uploaded``, ``sourceDatasetId``, - and ``size``. Follow ``cursor`` by passing it back as *after* while + ``size`` and, when one was recorded at upload, ``crc32`` (base64 + big-endian CRC-32; compare with :func:`ndi.fun.file.crc32`). Follow ``cursor`` by passing it back as *after* while ``hasMore`` is true. MATLAB equivalent: +cloud/+api/+files/listFiles.m @@ -459,6 +517,9 @@ def getFileDetails( ) -> dict[str, Any]: """Get detail info (including download URL) for a file. + The record carries ``crc32`` (base64 big-endian CRC-32 of the content) + when one was recorded at upload. + MATLAB equivalent: +cloud/+api/+files/getFileDetails.m """ return client.get( @@ -468,6 +529,59 @@ def getFileDetails( ) +@_auto_client +@validate_call(config=VALIDATE_CONFIG) +def lookupFiles( + dataset_id: CloudId, + file_uids: list[str], + *, + client: _Client = None, +) -> dict[str, Any]: + """Look up the upload state, size and CRC-32 of files by uid. + + ``POST /datasets/{datasetId}/files/lookup``. Tells, for each uid, whether + the dataset has the file, whether it finished uploading, its size, and + the CRC-32 recorded at upload, without listing the whole dataset or + signing any URLs. Use it to decide which files of a document or import + batch still need uploading, e.g. to resume an interrupted upload. + + The server takes at most :data:`LOOKUP_MAX_UIDS` (5000) uids per call; + longer lists are split into several calls and the results merged. + Duplicate uids are looked up once. + + Args: + dataset_id: The cloud dataset id. + file_uids: File uids to look up. + client: Authenticated cloud client (auto-created if omitted). + + Returns: + ``{"files": [...], "missing": [...]}``. ``files`` has one dict per uid + the dataset has a record for, with ``uid``, ``uploaded``, ``size`` and + ``crc32`` (base64 big-endian CRC-32, or ``None`` when none was + recorded -- compare ``size`` instead). ``missing`` lists the uids with + no record. + + A cloud copy matches a local file when it is uploaded and its ``crc32`` + equals :func:`ndi.fun.file.crc32` of the local file. + + MATLAB equivalent: +cloud/+api/+files/lookupFiles.m + """ + uids = list(dict.fromkeys(u for u in file_uids if u)) + result: dict[str, Any] = {"files": [], "missing": []} + for start in range(0, len(uids), LOOKUP_MAX_UIDS): + batch = uids[start : start + LOOKUP_MAX_UIDS] + page = client.post( + "/datasets/{datasetId}/files/lookup", + json={"uids": batch}, + datasetId=dataset_id, + ) + files = page.get("files", []) if hasattr(page, "get") else [] + missing = page.get("missing", []) if hasattr(page, "get") else [] + result["files"].extend(files or []) + result["missing"].extend(missing or []) + return result + + # --------------------------------------------------------------------------- # Signed-URL-set family # @@ -498,8 +612,10 @@ def getSignedURLSet( """GET one page of a document's uid -> signed URL map. Returns a dict with fields ``datasetId``, ``documentId``, ``totalCount``, - ``pageCount``, optionally ``nextCursor``, ``expiresAt``, and ``files`` - (a dict mapping file uid to pre-signed download URL, in uid-sorted order). + ``pageCount``, optionally ``nextCursor``, ``expiresAt``, ``files`` + (a dict mapping file uid to pre-signed download URL, in uid-sorted order) + and ``fileInfo`` (a list of ``{uid, uploaded, size, crc32}`` for the same + files; ``crc32`` is ``None`` when none was recorded). Args: dataset_id: The cloud dataset id. @@ -590,7 +706,8 @@ def getSignedURLSetAll( Returns: Merged dict with ``datasetId``, ``documentId``, ``files`` - (uid -> URL), ``pageCount`` (total uids across pages), ``pages`` + (uid -> URL), ``fileInfo`` (``{uid, uploaded, size, crc32}`` dicts + across pages), ``pageCount`` (total uids across pages), ``pages`` (number of HTTP calls), ``totalCount`` (server-reported total), and ``expiresAt`` (from the last page carrying it). @@ -605,6 +722,7 @@ def getSignedURLSetAll( "datasetId": dataset_id, "documentId": document_id, "files": {}, + "fileInfo": [], "pageCount": 0, "totalCount": 0, "pages": 0, @@ -624,6 +742,9 @@ def getSignedURLSetAll( if isinstance(files, dict): merged["files"].update(files) merged["pageCount"] += len(files) + file_info = page.get("fileInfo") if hasattr(page, "get") else None + if isinstance(file_info, list): + merged["fileInfo"].extend(file_info) total = page.get("totalCount") if hasattr(page, "get") else None if isinstance(total, int): merged["totalCount"] = total diff --git a/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml b/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml index 367b1ea0..4b587bd3 100644 --- a/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml +++ b/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml @@ -760,7 +760,7 @@ functions: # --- files.py --- - name: getFileUploadURL matlab_path: "+ndi/+cloud/+api/+files/getFileUploadURL.m" - matlab_last_sync_hash: "9b75c0fe" + matlab_last_sync_hash: "8fa6ea5e" python_path: "ndi/cloud/api/files.py" input_arguments: - name: org_id @@ -772,13 +772,22 @@ functions: - name: file_uid type_matlab: "char" type_python: "NonEmptyStr" + - name: checksum_crc32 + type_matlab: "string" + type_python: "str" + default: "''" - name: client type_python: "_Client" default: "None" output_arguments: - name: url type_python: "str" - decision_log: "Exact match." + decision_log: > + Exact match. 2026-10-12: synced to NDI-matlab 8fa6ea5e + (claude/file-crc32-lookup, NDI-matlab#1025): the 'checksumCRC32' + option becomes checksum_crc32, sent as the checksumCRC32 query + parameter (NDI Cloud ndi-cloud-node#162). The returned URL then signs + the x-amz-checksum-crc32 header, which putFiles must send. - name: getBulkUploadURL matlab_path: "+ndi/+cloud/+api/+documents/getBulkUploadURL.m" @@ -806,7 +815,7 @@ functions: - name: putFiles matlab_path: "+ndi/+cloud/+api/+files/putFiles.m" - matlab_last_sync_hash: "2566fe4d" + matlab_last_sync_hash: "8fa6ea5e" python_path: "ndi/cloud/api/files.py" input_arguments: - name: url @@ -830,10 +839,18 @@ functions: type_matlab: "double" type_python: "float" default: "60.0" + - name: checksum_crc32 + type_matlab: "string" + type_python: "str" + default: "''" output_arguments: - name: success type_python: "bool" decision_log: > + 2026-10-12: synced to NDI-matlab 8fa6ea5e (claude/file-crc32-lookup, + NDI-matlab#1025): 'checksumCRC32' becomes checksum_crc32 and is sent + as the x-amz-checksum-crc32 header, which a checksummed upload URL + signs (ndi-cloud-node#162). Validated as a base64 4-byte CRC-32. Synchronized with MATLAB main as of 2026-05-11 (matlab HEAD 2566fe4d). Added job_id and wait_for_completion options to mirror the new MATLAB bulk-upload "fire-and-wait" pattern: after a @@ -893,7 +910,7 @@ functions: - name: listFiles matlab_path: "+ndi/+cloud/+api/+files/listFiles.m" - matlab_last_sync_hash: "3fce6cb" + matlab_last_sync_hash: "8fa6ea5e" python_path: "ndi/cloud/api/files.py" input_arguments: - name: dataset_id @@ -925,7 +942,7 @@ functions: - name: listFilesAll matlab_path: "+ndi/+cloud/+api/+files/listFilesAll.m" - matlab_last_sync_hash: "3fce6cb" + matlab_last_sync_hash: "8fa6ea5e" python_path: "ndi/cloud/api/files.py" input_arguments: - name: dataset_id @@ -963,6 +980,31 @@ functions: MATLAB `retries` knob is omitted: transient GET retries are handled centrally by CloudClient, not per call site. + - name: lookupFiles + matlab_path: "+ndi/+cloud/+api/+files/lookupFiles.m" + matlab_last_sync_hash: "8fa6ea5e" + matlab_branch: "claude/file-crc32-lookup" + python_path: "ndi/cloud/api/files.py" + input_arguments: + - name: dataset_id + type_matlab: "string" + type_python: "CloudId" + - name: file_uids + type_matlab: "string array" + type_python: "list[str]" + - name: client + type_python: "_Client" + default: "None" + output_arguments: + - name: result + type_python: "dict[str, Any]" + decision_log: > + Exact match. POST /datasets/{datasetId}/files/lookup (ndi-cloud-node#162), + split into calls of at most LOOKUP_MAX_UIDS (5000) uids and merged, as + MATLAB does. Returns {"files": [{uid, uploaded, size, crc32}], + "missing": [uid, ...]}. Where MATLAB normalizes a missing crc32 to '' + (a struct field cannot be null), Python keeps the server's None. + - name: getFileDetails matlab_path: "+ndi/+cloud/+api/+files/getFileDetails.m" matlab_last_sync_hash: "eb577fbd" @@ -1639,7 +1681,7 @@ not_yet_ported: - name: url matlab_path: "+ndi/+cloud/+api/url.m" - matlab_last_sync_hash: "75f937a76" + matlab_last_sync_hash: "8fa6ea5e" status: ported_differently decision_log: > MATLAB URL builder. Python uses CloudClient._build_url() @@ -1671,6 +1713,11 @@ not_yet_ported: get_file_tier_job -> GET /file-tier-jobs/{jobId}); both are inlined at the call sites in api/files.py by setFileTier / getFileTierJob, so this entry has nothing to add. + 2026-10-12: examined against 8fa6ea5e (was 75f937a76), NDI-matlab#1025: + one row added to url()'s route table, lookup_dataset_files -> + /datasets/{datasetId}/files/lookup. Nothing to port here; the route + string lives in ndi.cloud.api.files.lookupFiles, as every other route + does in Python. - name: documentCount matlab_path: "+ndi/+cloud/+api/+documents/documentCount.m" @@ -1720,7 +1767,7 @@ not_yet_ported: - name: getSignedURLSet matlab_path: "+ndi/+cloud/+api/+files/getSignedURLSet.m" python_path: "ndi/cloud/api/files.py" - matlab_last_sync_hash: "5804cd80" + matlab_last_sync_hash: "8fa6ea5e" status: ported decision_log: > Ported by NDI-python#262 as the API foundation of batch_signed_url. @@ -1735,7 +1782,7 @@ not_yet_ported: - name: getSignedURLSetAll matlab_path: "+ndi/+cloud/+api/+files/getSignedURLSetAll.m" python_path: "ndi/cloud/api/files.py" - matlab_last_sync_hash: "5804cd80" + matlab_last_sync_hash: "8fa6ea5e" status: ported decision_log: > Ported by NDI-python#262. Walks getSignedURLSet cursors and merges @@ -1746,6 +1793,10 @@ not_yet_ported: partial merged dict on the exception. The async job path (createSignedURLSetJob + waitForSignedURLSetJob) is still deferred because batch_signed_url does not use it. + 2026-10-12: synced to NDI-matlab 8fa6ea5e (NDI-matlab#1025): pages' + fileInfo lists ({uid, uploaded, size, crc32}, ndi-cloud-node#162) are + concatenated into merged["fileInfo"]. getSignedURLSet passes fileInfo + through untouched. - name: getSignedURLSetJob matlab_path: "+ndi/+cloud/+api/+files/getSignedURLSetJob.m" diff --git a/src/ndi/cloud/ndi_matlab_python_bridge.yaml b/src/ndi/cloud/ndi_matlab_python_bridge.yaml index 77f35ad2..47731d6b 100644 --- a/src/ndi/cloud/ndi_matlab_python_bridge.yaml +++ b/src/ndi/cloud/ndi_matlab_python_bridge.yaml @@ -967,7 +967,7 @@ functions: - name: uploadSingleFile matlab_path: "+ndi/+cloud/uploadSingleFile.m" - matlab_last_sync_hash: "d9213ccb7" + matlab_last_sync_hash: "8fa6ea5e" python_path: "ndi/cloud/upload.py" input_arguments: - name: dataset_id @@ -983,6 +983,10 @@ functions: type_matlab: "logical" type_python: "bool" default: "False" + - name: checksum + type_matlab: "logical" + type_python: "bool" + default: "True" - name: client type_python: "CloudClient | None" default: "None" @@ -993,6 +997,10 @@ functions: type_python: "str" decision_log: > Exact match. + 2026-10-12: synced to NDI-matlab 8fa6ea5e (claude/file-crc32-lookup, + NDI-matlab#1025): 'checksum' (default true) computes the file's CRC-32 + and sends it with the single-file upload URL request and the PUT + (ndi-cloud-node#162). 2026-09-08: examined against d9213ccb7 (was 7026d493). Two commits. 39ac4cabe forwards useCurl through the bulk branch and does not port. d9213ccb7 adds the bulk-upload status API and a waitForCompletion option on diff --git a/src/ndi/cloud/upload.py b/src/ndi/cloud/upload.py index fad86a54..9c173831 100644 --- a/src/ndi/cloud/upload.py +++ b/src/ndi/cloud/upload.py @@ -369,6 +369,7 @@ def uploadSingleFile( file_path: str, *, use_bulk_upload: bool = False, + checksum: bool = True, client: CloudClient | None = None, ) -> tuple[bool, str]: """Upload a single file to the NDI cloud service. @@ -381,6 +382,11 @@ def uploadSingleFile( file_path: Local path of the file to upload. use_bulk_upload: If True, zip the file and use the bulk upload mechanism. Defaults to False. + checksum: If True (default), compute the file's CRC-32 + (:func:`ndi.fun.file.crc32`) and send it with the single-file + upload, so S3 rejects a corrupted upload and NDI Cloud records + the file's ``crc32``. Ignored for bulk uploads, whose zip entries + carry their own CRC-32. client: Authenticated cloud client (auto-created if omitted). Returns: @@ -408,13 +414,19 @@ def uploadSingleFile( if zip_path.exists(): zip_path.unlink() else: + checksum_crc32 = "" + if checksum: + from ..fun.file import crc32 + + checksum_crc32 = crc32(file_path) url = files_api.getFileUploadURL( client.config.org_id, dataset_id, file_uid, + checksum_crc32=checksum_crc32, client=client, ) - files_api.putFiles(url, file_path) + files_api.putFiles(url, file_path, checksum_crc32=checksum_crc32) return True, "" except Exception as exc: diff --git a/src/ndi/fun/file.py b/src/ndi/fun/file.py index eb081d65..724bc2fd 100644 --- a/src/ndi/fun/file.py +++ b/src/ndi/fun/file.py @@ -1,12 +1,15 @@ """ ndi.fun.file - File utility functions. -MATLAB equivalents: +ndi/+fun/+file/MD5.m, dateCreated.m, dateUpdated.m +MATLAB equivalents: +ndi/+fun/+file/MD5.m, crc32.m, crc32ToBase64.m, + dateCreated.m, dateUpdated.m """ from __future__ import annotations +import base64 import hashlib +import zlib from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -36,6 +39,60 @@ def MD5(file_path: str) -> str: return h.hexdigest() +def crc32ToBase64(value: int) -> str: + """Encode a numeric CRC-32 as base64 of its 4-byte big-endian value. + + MATLAB equivalent: ndi.fun.file.crc32ToBase64 + + NDI Cloud and S3 represent a CRC-32 this way (``x-amz-checksum-crc32``). + + Args: + value: The CRC-32, an integer in ``[0, 0xFFFFFFFF]`` (e.g. from + ``zlib.crc32``). + + Returns: + 8-character base64 string, e.g. ``'NhCmhg=='`` for ``0x3610A686``. + + Raises: + ValueError: If *value* is not a 32-bit unsigned integer. + """ + if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= 0xFFFFFFFF: + raise ValueError(f"CRC-32 must be an integer in [0, 0xFFFFFFFF], got {value!r}") + return base64.b64encode(value.to_bytes(4, "big")).decode("ascii") + + +def crc32(file_path: str, chunk_size: int = 8 * 1024 * 1024) -> str: + """CRC-32 of a file, base64-encoded the way NDI Cloud and S3 store it. + + MATLAB equivalent: ndi.fun.file.crc32 + + The CRC-32 (IEEE, as ``zlib.crc32``) of the file's bytes, returned as the + base64 of its 4-byte big-endian value. That is the value + :func:`ndi.cloud.api.files.getFileUploadURL` sends as ``checksum_crc32`` + and the value NDI Cloud returns as a file's ``crc32``. The file is read in + chunks. + + Args: + file_path: Path to the file. + chunk_size: Bytes read per chunk. Default 8 MiB. + + Returns: + 8-character base64 string, e.g. ``'NhCmhg=='`` for a file containing + ``b'hello'``. + + Raises: + FileNotFoundError: If the file does not exist. + """ + p = Path(file_path) + if not p.is_file(): + raise FileNotFoundError(f"File not found: {file_path}") + value = 0 + with open(p, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + value = zlib.crc32(chunk, value) + return crc32ToBase64(value) + + def dateCreated(file_path: str) -> datetime | None: """Get the creation date of a file. diff --git a/src/ndi/fun/ndi_matlab_python_bridge.yaml b/src/ndi/fun/ndi_matlab_python_bridge.yaml index 540ee534..68eb096c 100644 --- a/src/ndi/fun/ndi_matlab_python_bridge.yaml +++ b/src/ndi/fun/ndi_matlab_python_bridge.yaml @@ -1298,6 +1298,46 @@ functions: the 32-character lowercase digest and both refuse a missing file. Nothing to port. + - name: crc32 + type: function + matlab_path: "+ndi/+fun/+file/crc32.m" + matlab_last_sync_hash: "8fa6ea5e" + matlab_branch: "claude/file-crc32-lookup" + python_path: "ndi/fun/file.py" + input_arguments: + - name: file_path + type_matlab: "string (fileName)" + type_python: "str" + - name: chunk_size + type_matlab: "double (chunkSize)" + type_python: "int" + default: "8388608" + output_arguments: + - name: checksum + type_python: "str" + decision_log: > + Exact match. CRC-32 (IEEE) of the file's bytes as base64 of the 4-byte + big-endian value, the encoding NDI Cloud and S3 use + (ndi-cloud-node#162). MATLAB uses java.util.zip.CRC32, Python + zlib.crc32; both read in chunks and give the same value. + + - name: crc32ToBase64 + type: function + matlab_path: "+ndi/+fun/+file/crc32ToBase64.m" + matlab_last_sync_hash: "8fa6ea5e" + matlab_branch: "claude/file-crc32-lookup" + python_path: "ndi/fun/file.py" + input_arguments: + - name: value + type_matlab: "double" + type_python: "int" + output_arguments: + - name: checksum + type_python: "str" + decision_log: > + Exact match. Base64 of the 4-byte big-endian value; rejects values + outside [0, 0xFFFFFFFF]. + - name: dateCreated type: function matlab_path: "+ndi/+fun/+file/dateCreated.m" diff --git a/tests/test_cloud_api_files_crc32.py b/tests/test_cloud_api_files_crc32.py new file mode 100644 index 00000000..111497cf --- /dev/null +++ b/tests/test_cloud_api_files_crc32.py @@ -0,0 +1,272 @@ +"""Unit tests for file CRC-32 support (NDI Cloud ndi-cloud-node#162) -- no network. + +MATLAB counterpart: ``tests/+ndi/+unittest/FileCrc32OfflineTest.m`` +(VH-Lab/NDI-matlab#1025). + +* :func:`ndi.fun.file.crc32` gives the cloud's encoding: base64 of the + 4-byte big-endian CRC-32. Expected values are zlib reference values. +* ``getFileUploadURL(..., checksum_crc32=...)`` sends ``checksumCRC32``. +* ``putFiles`` / ``putFileBytes`` send the signed ``x-amz-checksum-crc32`` + header, and refuse a malformed value. +* ``lookupFiles`` posts ``{"uids": [...]}``, de-duplicates, splits at the + server's 5000-uid limit and merges ``files`` / ``missing``. +* ``getSignedURLSetAll`` concatenates the pages' ``fileInfo``. +* ``uploadSingleFile`` sends the checksum by default. + +The live round trip is ``TestFileLifecycle`` in ``tests/test_cloud_live.py``. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from ndi.cloud.api import files as files_api +from ndi.fun.file import crc32, crc32ToBase64 + +DATASET = "507f1f77bcf86cd799439011" + + +# --------------------------------------------------------------------------- +# Encoding +# --------------------------------------------------------------------------- + + +class TestCrc32Encoding: + def test_big_endian_base64(self): + assert crc32ToBase64(0x3610A686) == "NhCmhg==" + assert crc32ToBase64(0) == "AAAAAA==" + assert crc32ToBase64(0xFFFFFFFF) == "/////w==" + + @pytest.mark.parametrize("bad", [-1, 2**32, 1.5, True, "NhCmhg=="]) + def test_rejects_values_outside_32_bits(self, bad): + with pytest.raises(ValueError): + crc32ToBase64(bad) + + def test_file_crc_of_known_text(self, tmp_path): + p = tmp_path / "f.bin" + p.write_bytes(b"hello") + assert crc32(str(p)) == "NhCmhg==" + + def test_empty_file(self, tmp_path): + p = tmp_path / "f.bin" + p.write_bytes(b"") + assert crc32(str(p)) == "AAAAAA==" + + def test_chunk_size_does_not_matter(self, tmp_path): + p = tmp_path / "f.bin" + p.write_bytes(b"The quick brown fox jumps over the lazy dog") + assert crc32(str(p)) == "QU+jOQ==" + assert crc32(str(p), chunk_size=1) == "QU+jOQ==" + assert crc32(str(p), chunk_size=7) == "QU+jOQ==" + + def test_many_chunks_and_high_bytes(self, tmp_path): + p = tmp_path / "f.bin" + p.write_bytes(bytes(range(256)) * 1000) + assert crc32(str(p), chunk_size=4096) == "/HCvGg==" + + def test_missing_file(self, tmp_path): + with pytest.raises(FileNotFoundError): + crc32(str(tmp_path / "nope")) + + +# --------------------------------------------------------------------------- +# Upload URL and PUT +# --------------------------------------------------------------------------- + + +class TestChecksummedUpload: + def test_upload_url_sends_checksum_query(self): + client = MagicMock() + client.get.return_value = {"url": "https://s3/x"} + + url = files_api.getFileUploadURL( + "org1", DATASET, "uid1", checksum_crc32="QU+jOQ==", client=client + ) + + assert url == "https://s3/x" + assert client.get.call_args.kwargs["params"] == {"checksumCRC32": "QU+jOQ=="} + + def test_upload_url_without_checksum_sends_no_query(self): + client = MagicMock() + client.get.return_value = {"url": "https://s3/x"} + + files_api.getFileUploadURL("org1", DATASET, "uid1", client=client) + + assert client.get.call_args.kwargs["params"] is None + + def test_upload_url_refuses_a_malformed_checksum(self): + with pytest.raises(ValueError): + files_api.getFileUploadURL( + "org1", DATASET, "uid1", checksum_crc32="abc", client=MagicMock() + ) + + def test_put_sends_the_checksum_header(self, tmp_path): + p = tmp_path / "f.bin" + p.write_bytes(b"hello") + resp = MagicMock(status_code=200) + with patch("requests.put", return_value=resp) as put: + assert files_api.putFiles( + "https://bucket.s3.amazonaws.com/k", str(p), checksum_crc32="NhCmhg==" + ) + headers = put.call_args.kwargs["headers"] + assert headers["x-amz-checksum-crc32"] == "NhCmhg==" + assert headers["Content-Type"] == "application/octet-stream" + + def test_put_without_checksum_sends_no_header(self, tmp_path): + p = tmp_path / "f.bin" + p.write_bytes(b"hello") + with patch("requests.put", return_value=MagicMock(status_code=200)) as put: + files_api.putFiles("https://bucket.s3.amazonaws.com/k", str(p)) + assert "x-amz-checksum-crc32" not in put.call_args.kwargs["headers"] + + def test_put_refuses_a_malformed_checksum(self, tmp_path): + p = tmp_path / "f.bin" + p.write_bytes(b"hello") + with patch("requests.put") as put, pytest.raises(ValueError): + files_api.putFiles( + "https://bucket.s3.amazonaws.com/k", str(p), checksum_crc32="not a crc" + ) + put.assert_not_called() + + def test_put_bytes_sends_the_checksum_header(self): + with patch("requests.put", return_value=MagicMock(status_code=200)) as put: + files_api.putFileBytes( + "https://bucket.s3.amazonaws.com/k", b"hello", checksum_crc32="NhCmhg==" + ) + assert put.call_args.kwargs["headers"]["x-amz-checksum-crc32"] == "NhCmhg==" + + +# --------------------------------------------------------------------------- +# lookupFiles +# --------------------------------------------------------------------------- + + +class _LookupServer: + """A fake POST /files/lookup over an in-memory {uid: record} map.""" + + def __init__(self, records): + self.records = records + self.calls = [] + + def post(self, endpoint, json=None, data=None, **path_params): + self.calls.append({"endpoint": endpoint, "json": json, "path": path_params}) + uids = json["uids"] + assert len(uids) <= files_api.LOOKUP_MAX_UIDS, "client must split long lists" + return { + "files": [self.records[u] for u in uids if u in self.records], + "missing": [u for u in uids if u not in self.records], + } + + +class TestLookupFiles: + def test_found_and_missing(self): + server = _LookupServer( + { + "u1": {"uid": "u1", "uploaded": True, "size": 5, "crc32": "NhCmhg=="}, + "u2": {"uid": "u2", "uploaded": False, "size": 0, "crc32": None}, + } + ) + + result = files_api.lookupFiles(DATASET, ["u1", "u2", "u3", "u1"], client=server) + + assert len(server.calls) == 1 + call = server.calls[0] + assert call["endpoint"] == "/datasets/{datasetId}/files/lookup" + assert call["path"] == {"datasetId": DATASET} + assert call["json"] == {"uids": ["u1", "u2", "u3"]} + assert [f["uid"] for f in result["files"]] == ["u1", "u2"] + assert result["files"][0]["crc32"] == "NhCmhg==" + assert result["files"][1]["crc32"] is None + assert result["missing"] == ["u3"] + + def test_splits_lists_over_the_server_limit(self): + server = _LookupServer({"u0": {"uid": "u0", "uploaded": True, "size": 1, "crc32": None}}) + uids = [f"u{i}" for i in range(files_api.LOOKUP_MAX_UIDS + 3)] + + result = files_api.lookupFiles(DATASET, uids, client=server) + + assert [len(c["json"]["uids"]) for c in server.calls] == [files_api.LOOKUP_MAX_UIDS, 3] + assert [f["uid"] for f in result["files"]] == ["u0"] + assert len(result["missing"]) == len(uids) - 1 + + def test_no_uids_makes_no_call(self): + server = _LookupServer({}) + assert files_api.lookupFiles(DATASET, [], client=server) == {"files": [], "missing": []} + assert server.calls == [] + + +# --------------------------------------------------------------------------- +# signed-url-set fileInfo +# --------------------------------------------------------------------------- + + +class TestSignedURLSetFileInfo: + def test_all_concatenates_file_info_across_pages(self): + pages = [ + { + "files": {"4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6": "https://s3/a"}, + "fileInfo": [ + { + "uid": "4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6", + "uploaded": True, + "size": 5, + "crc32": "NhCmhg==", + } + ], + "nextCursor": "c1", + }, + { + "files": {"b": "https://s3/b"}, + "fileInfo": [{"uid": "b", "uploaded": True, "size": 7, "crc32": None}], + }, + ] + client = MagicMock() + client.get.side_effect = pages + + merged = files_api.getSignedURLSetAll(DATASET, "doc1", client=client) + + assert [f["uid"] for f in merged["fileInfo"]] == [ + "4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6", + "b", + ] + assert merged["fileInfo"][0]["crc32"] == "NhCmhg==" + + def test_all_without_file_info_gives_an_empty_list(self): + client = MagicMock() + client.get.return_value = {"files": {"a": "https://s3/a"}} + merged = files_api.getSignedURLSetAll(DATASET, "doc1", client=client) + assert merged["fileInfo"] == [] + + +# --------------------------------------------------------------------------- +# uploadSingleFile +# --------------------------------------------------------------------------- + + +class TestUploadSingleFileChecksum: + def _run(self, tmp_path, **kwargs): + from ndi.cloud.upload import uploadSingleFile + + p = tmp_path / "f.bin" + p.write_bytes(b"hello") + client = MagicMock() + client.config.org_id = "org1" + with ( + patch.object(files_api, "getFileUploadURL", return_value="https://s3/x") as get_url, + patch.object(files_api, "putFiles", return_value=True) as put, + ): + ok, err = uploadSingleFile(DATASET, "uid1", str(p), client=client, **kwargs) + assert ok, err + return get_url, put + + def test_sends_the_checksum_by_default(self, tmp_path): + get_url, put = self._run(tmp_path) + assert get_url.call_args.kwargs["checksum_crc32"] == "NhCmhg==" + assert put.call_args.kwargs["checksum_crc32"] == "NhCmhg==" + + def test_checksum_can_be_turned_off(self, tmp_path): + get_url, put = self._run(tmp_path, checksum=False) + assert get_url.call_args.kwargs["checksum_crc32"] == "" + assert put.call_args.kwargs["checksum_crc32"] == "" diff --git a/tests/test_cloud_live.py b/tests/test_cloud_live.py index 0a16f1d3..c7b80848 100644 --- a/tests/test_cloud_live.py +++ b/tests/test_cloud_live.py @@ -796,6 +796,145 @@ def test_file_details_has_download_url(self, client, fresh_dataset, cloud_config assert details.get("downloadUrl") +# =========================================================================== +# TestFileChecksums -- matches MATLAB FilesTest checksum / lookupFiles tests +# (NDI Cloud ndi-cloud-node#162, NDI-matlab#1025) +# =========================================================================== + + +def _skip_unless_server_has_file_lookup(client, dataset_id): + """Skip when this environment predates POST /files/lookup (#162). + + The route and the crc32 recording ship together; a 404 on a dataset that + exists means the server has neither yet. + """ + from ndi.cloud.api.files import lookupFiles + from ndi.cloud.exceptions import CloudNotFoundError + + try: + lookupFiles(dataset_id, ["probe_uid"], client=client) + except CloudNotFoundError: + pytest.skip("This NDI Cloud environment predates POST /files/lookup (#162)") + + +def _wait_for_lookup(client, dataset_id, uid, missing_uid=None): + """Poll lookupFiles until *uid* is uploaded (the S3 event is async).""" + from ndi.cloud.api.files import lookupFiles + + uids = [uid] + ([missing_uid] if missing_uid else []) + result = {} + for wait in (3, 5, 10, 15): + time.sleep(wait) + result = lookupFiles(dataset_id, uids, client=client) + found = [f for f in result["files"] if f.get("uid") == uid] + if found and found[0].get("uploaded"): + break + return result + + +class TestFileChecksums: + CONTENT = b"Checksum test file for NDI Cloud #162." + + def test_upload_with_checksum_round_trip(self, client, cloud_config, fresh_dataset, tmp_path): + """A checksum sent with the upload is returned as crc32 everywhere.""" + from ndi.cloud.api.files import ( + getFileDetails, + getFileUploadURL, + listFiles, + putFiles, + ) + from ndi.fun.file import crc32 + + _skip_unless_server_has_file_lookup(client, fresh_dataset) + path = tmp_path / "payload.bin" + path.write_bytes(self.CONTENT) + local_crc = crc32(str(path)) + file_uid = "pytest-crc32-roundtrip" + missing_uid = "pytest-crc32-never-uploaded" + + url = getFileUploadURL( + cloud_config.org_id, fresh_dataset, file_uid, checksum_crc32=local_crc, client=client + ) + assert putFiles(url, str(path), checksum_crc32=local_crc) + + result = _wait_for_lookup(client, fresh_dataset, file_uid, missing_uid) + assert [f["uid"] for f in result["files"]] == [file_uid] + rec = result["files"][0] + assert rec["uploaded"] is True + assert rec["size"] == len(self.CONTENT) + assert rec["crc32"] == local_crc + assert result["missing"] == [missing_uid] + + details = getFileDetails(fresh_dataset, file_uid, client=client) + assert details.get("crc32") == local_crc + + rows = list(listFiles(fresh_dataset, client=client).get("files", [])) + row = [f for f in rows if f.get("uid") == file_uid] + assert row and row[0].get("crc32") == local_crc + + def test_upload_with_wrong_checksum_is_rejected( + self, client, cloud_config, fresh_dataset, tmp_path + ): + """S3 refuses content whose CRC-32 does not match the signed checksum.""" + from ndi.cloud.api.files import getFileUploadURL, putFiles + from ndi.cloud.exceptions import CloudUploadError + from ndi.fun.file import crc32, crc32ToBase64 + + _skip_unless_server_has_file_lookup(client, fresh_dataset) + path = tmp_path / "payload.bin" + path.write_bytes(self.CONTENT) + wrong = crc32ToBase64(0) + assert crc32(str(path)) != wrong + + url = getFileUploadURL( + cloud_config.org_id, + fresh_dataset, + "pytest-crc32-wrong", + checksum_crc32=wrong, + client=client, + ) + with pytest.raises(CloudUploadError): + putFiles(url, str(path), checksum_crc32=wrong) + + def test_upload_single_file_records_checksum(self, client, fresh_dataset, tmp_path): + """ndi.cloud.upload.uploadSingleFile sends the checksum by default.""" + from ndi.cloud.upload import uploadSingleFile + from ndi.fun.file import crc32 + + _skip_unless_server_has_file_lookup(client, fresh_dataset) + path = tmp_path / "payload.bin" + path.write_bytes(self.CONTENT) + file_uid = "pytest-crc32-single" + + ok, err = uploadSingleFile(fresh_dataset, file_uid, str(path), client=client) + assert ok, err + + result = _wait_for_lookup(client, fresh_dataset, file_uid) + assert result["files"] and result["files"][0]["crc32"] == crc32(str(path)) + + def test_lookup_files_rejects_an_invalid_body(self, client, fresh_dataset): + """The server refuses more than 5000 uids, or a body without uids.""" + from ndi.cloud.exceptions import CloudAPIError + + _skip_unless_server_has_file_lookup(client, fresh_dataset) + for body in ({"uids": [f"u{i}" for i in range(5001)]}, {"notUids": 1}): + with pytest.raises(CloudAPIError) as exc_info: + client.post( + "/datasets/{datasetId}/files/lookup", json=body, datasetId=fresh_dataset + ) + assert exc_info.value.status_code == 400 + + def test_lookup_files_splits_long_lists(self, client, fresh_dataset): + """More than one server call's worth of uids, none uploaded.""" + from ndi.cloud.api.files import LOOKUP_MAX_UIDS, lookupFiles + + _skip_unless_server_has_file_lookup(client, fresh_dataset) + uids = [f"nouid{i}" for i in range(LOOKUP_MAX_UIDS + 3)] + result = lookupFiles(fresh_dataset, uids, client=client) + assert result["files"] == [] + assert sorted(result["missing"]) == sorted(uids) + + # =========================================================================== # TestNDIQuery -- matches MATLAB testNdiQuery # =========================================================================== diff --git a/tests/test_phase2_gaps.py b/tests/test_phase2_gaps.py index d200ec11..eab60d5a 100644 --- a/tests/test_phase2_gaps.py +++ b/tests/test_phase2_gaps.py @@ -440,8 +440,11 @@ def test_direct_upload_success(self): with patch.dict("sys.modules", {"ndi.cloud.api.files": mock_files}): with patch("ndi.cloud.api.files", mock_files): + # The file is never read here (the upload API is mocked), so + # skip the CRC-32 that uploadSingleFile computes by default; + # tests/test_cloud_api_files_crc32.py covers that path. success, err = uploadSingleFile( - "ds-123", "file-uid-1", "/tmp/test.dat", client=client + "ds-123", "file-uid-1", "/tmp/test.dat", checksum=False, client=client ) assert success is True @@ -457,8 +460,10 @@ def test_upload_failure(self): with patch.dict("sys.modules", {"ndi.cloud.api.files": mock_files}): with patch("ndi.cloud.api.files", mock_files): + # As above: the file does not exist, so skip the default CRC-32 + # and let the mocked network error be what fails. success, err = uploadSingleFile( - "ds-123", "file-uid-1", "/tmp/test.dat", client=client + "ds-123", "file-uid-1", "/tmp/test.dat", checksum=False, client=client ) assert success is False From e7ca3b021094efa45853f6a4feb6d772ff879d51 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 23:54:52 +0000 Subject: [PATCH 2/3] Fail, not skip, live checksum tests on a server without #162 A skip hid whether the tests ran at all; an environment without the lookup route or fileInfo now fails these tests. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SwsqFA9Zs1HGm6YM4g2XRV --- tests/test_cloud_live.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tests/test_cloud_live.py b/tests/test_cloud_live.py index c7b80848..c98089ff 100644 --- a/tests/test_cloud_live.py +++ b/tests/test_cloud_live.py @@ -802,21 +802,6 @@ def test_file_details_has_download_url(self, client, fresh_dataset, cloud_config # =========================================================================== -def _skip_unless_server_has_file_lookup(client, dataset_id): - """Skip when this environment predates POST /files/lookup (#162). - - The route and the crc32 recording ship together; a 404 on a dataset that - exists means the server has neither yet. - """ - from ndi.cloud.api.files import lookupFiles - from ndi.cloud.exceptions import CloudNotFoundError - - try: - lookupFiles(dataset_id, ["probe_uid"], client=client) - except CloudNotFoundError: - pytest.skip("This NDI Cloud environment predates POST /files/lookup (#162)") - - def _wait_for_lookup(client, dataset_id, uid, missing_uid=None): """Poll lookupFiles until *uid* is uploaded (the S3 event is async).""" from ndi.cloud.api.files import lookupFiles @@ -845,7 +830,6 @@ def test_upload_with_checksum_round_trip(self, client, cloud_config, fresh_datas ) from ndi.fun.file import crc32 - _skip_unless_server_has_file_lookup(client, fresh_dataset) path = tmp_path / "payload.bin" path.write_bytes(self.CONTENT) local_crc = crc32(str(path)) @@ -880,7 +864,6 @@ def test_upload_with_wrong_checksum_is_rejected( from ndi.cloud.exceptions import CloudUploadError from ndi.fun.file import crc32, crc32ToBase64 - _skip_unless_server_has_file_lookup(client, fresh_dataset) path = tmp_path / "payload.bin" path.write_bytes(self.CONTENT) wrong = crc32ToBase64(0) @@ -901,7 +884,6 @@ def test_upload_single_file_records_checksum(self, client, fresh_dataset, tmp_pa from ndi.cloud.upload import uploadSingleFile from ndi.fun.file import crc32 - _skip_unless_server_has_file_lookup(client, fresh_dataset) path = tmp_path / "payload.bin" path.write_bytes(self.CONTENT) file_uid = "pytest-crc32-single" @@ -916,7 +898,6 @@ def test_lookup_files_rejects_an_invalid_body(self, client, fresh_dataset): """The server refuses more than 5000 uids, or a body without uids.""" from ndi.cloud.exceptions import CloudAPIError - _skip_unless_server_has_file_lookup(client, fresh_dataset) for body in ({"uids": [f"u{i}" for i in range(5001)]}, {"notUids": 1}): with pytest.raises(CloudAPIError) as exc_info: client.post( @@ -928,7 +909,6 @@ def test_lookup_files_splits_long_lists(self, client, fresh_dataset): """More than one server call's worth of uids, none uploaded.""" from ndi.cloud.api.files import LOOKUP_MAX_UIDS, lookupFiles - _skip_unless_server_has_file_lookup(client, fresh_dataset) uids = [f"nouid{i}" for i in range(LOOKUP_MAX_UIDS + 3)] result = lookupFiles(fresh_dataset, uids, client=client) assert result["files"] == [] From d469c5b5e97f365e58221b018d5eccec07cb430b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 12:57:27 +0000 Subject: [PATCH 3/3] Drop matlab_branch markers now that NDI-matlab#1025 has merged lookupFiles, crc32 and crc32ToBase64 are on NDI-matlab main, so the ordinary bridge checks apply. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SwsqFA9Zs1HGm6YM4g2XRV --- src/ndi/cloud/api/ndi_matlab_python_bridge.yaml | 1 - src/ndi/fun/ndi_matlab_python_bridge.yaml | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml b/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml index 4b587bd3..8b6131b6 100644 --- a/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml +++ b/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml @@ -983,7 +983,6 @@ functions: - name: lookupFiles matlab_path: "+ndi/+cloud/+api/+files/lookupFiles.m" matlab_last_sync_hash: "8fa6ea5e" - matlab_branch: "claude/file-crc32-lookup" python_path: "ndi/cloud/api/files.py" input_arguments: - name: dataset_id diff --git a/src/ndi/fun/ndi_matlab_python_bridge.yaml b/src/ndi/fun/ndi_matlab_python_bridge.yaml index 68eb096c..5edc5867 100644 --- a/src/ndi/fun/ndi_matlab_python_bridge.yaml +++ b/src/ndi/fun/ndi_matlab_python_bridge.yaml @@ -1302,7 +1302,6 @@ functions: type: function matlab_path: "+ndi/+fun/+file/crc32.m" matlab_last_sync_hash: "8fa6ea5e" - matlab_branch: "claude/file-crc32-lookup" python_path: "ndi/fun/file.py" input_arguments: - name: file_path @@ -1325,7 +1324,6 @@ functions: type: function matlab_path: "+ndi/+fun/+file/crc32ToBase64.m" matlab_last_sync_hash: "8fa6ea5e" - matlab_branch: "claude/file-crc32-lookup" python_path: "ndi/fun/file.py" input_arguments: - name: value