From 05d9b341e501794ac8ddfc8ca5b01570b5d8c0bd Mon Sep 17 00:00:00 2001
From: Claudio Cannizzaro <claudio.cannizzaro@fgcz.uzh.ch>
Date: Tue, 29 Sep 2026 11:44:58 +0200
Subject: [PATCH 01/12] feat: skip browser on remote login, show MD5 progress
 in workunit upload

pkce_login prints the login URL instead of opening a browser over SSH, without a
display, or with only a text browser. upload_files gains on_hash_progress, shown as
a bar by bfabric-cli workunit upload.
---
 bfabric/docs/changelog.md                     |  8 ++++
 bfabric/src/bfabric/oauth/_pkce.py            | 26 ++++++++++-
 .../src/bfabric/operations/workunit/upload.py | 13 ++++--
 .../bfabric/transfer/_generic/checksums.py    | 43 +++++++++++++++----
 .../bfabric_scripts/cli/login/oauth_login.py  |  1 -
 .../bfabric_scripts/cli/workunit/upload.py    | 11 +++++
 tests/bfabric/oauth/test_pkce.py              | 41 ++++++++++++++++++
 .../transfer/generic/test_checksums.py        | 22 ++++++++++
 8 files changed, 151 insertions(+), 14 deletions(-)

diff --git a/bfabric/docs/changelog.md b/bfabric/docs/changelog.md
index 7f8ad0e59..7be583e24 100644
--- a/bfabric/docs/changelog.md
+++ b/bfabric/docs/changelog.md
@@ -9,6 +9,14 @@ Minor breaking changes are still possible in `1.X.Y` but we try to announce them
 
 ## \[Unreleased\]
 
+### Added
+
+- `upload_files` accepts `on_hash_progress`, called while each file's MD5 is computed; `bfabric-cli workunit upload` shows it as a progress bar.
+
+### Changed
+
+- `pkce_login` no longer tries to open a browser over SSH, without a display, or when only a text browser is available; it prints the login URL instead.
+
 ## \[1.23.0\] - 2026-09-16
 
 ### Added
diff --git a/bfabric/src/bfabric/oauth/_pkce.py b/bfabric/src/bfabric/oauth/_pkce.py
index 24ee7677d..305fcbf71 100644
--- a/bfabric/src/bfabric/oauth/_pkce.py
+++ b/bfabric/src/bfabric/oauth/_pkce.py
@@ -11,11 +11,13 @@
 import base64
 import hashlib
 import html
+import os
 import secrets
 import sys
 import threading
 import webbrowser
 from dataclasses import dataclass
+from pathlib import Path
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from typing import TYPE_CHECKING, ClassVar
 from urllib.parse import parse_qs, urlparse
@@ -227,6 +229,25 @@ def exchange_code(
     return result
 
 
+_TEXT_BROWSERS = frozenset({"www-browser", "links", "links2", "elinks", "lynx", "w3m", "browsh"})
+
+
+def _graphical_browser_available() -> bool:
+    """Whether opening a browser here would plausibly reach a GUI the user can see.
+
+    ``webbrowser.open`` reports success for a text browser or a remote session, which only strands the user.
+    """
+    if os.environ.get("SSH_CONNECTION") or os.environ.get("SSH_TTY"):
+        return False
+    if sys.platform.startswith("linux") and not (os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY")):
+        return False
+    try:
+        name = getattr(webbrowser.get(), "name", "")
+    except webbrowser.Error:
+        return False
+    return Path(name).name not in _TEXT_BROWSERS
+
+
 def pkce_login(
     base_url: BaseUrl,
     *,
@@ -239,7 +260,8 @@ def pkce_login(
     """Perform an OAuth 2.0 Authorization Code flow with PKCE.
 
     :param port: Local port for the callback server (``0`` = auto-assign)
-    :param open_browser: If ``False``, or if the browser fails to open, the URL is printed to stderr
+    :param open_browser: If ``False``, over SSH, without a display or with only a text browser, or if the browser
+        fails to open, the URL is printed to stderr instead
     :param timeout: Seconds to wait for the user to complete login
     :returns: Token dict with ``access_token``, ``refresh_token``, etc.
     :raises BfabricOAuthError: On timeout, CSRF state mismatch, or authorization error
@@ -255,7 +277,7 @@ def pkce_login(
     )
 
     browser_opened = False
-    if open_browser:
+    if open_browser and _graphical_browser_available():
         browser_opened = webbrowser.open(request.url)
     if not browser_opened:
         print(
diff --git a/bfabric/src/bfabric/operations/workunit/upload.py b/bfabric/src/bfabric/operations/workunit/upload.py
index 27895f6af..7ca67c5e3 100644
--- a/bfabric/src/bfabric/operations/workunit/upload.py
+++ b/bfabric/src/bfabric/operations/workunit/upload.py
@@ -188,6 +188,7 @@ def upload_files(
     params: UploadFilesParams,
     *,
     on_progress: FileProgressCallback | None = None,
+    on_hash_progress: FileProgressCallback | None = None,
     on_start: UploadStartCallback | None = None,
     on_file_done: FileDoneCallback | None = None,
     on_url: FileUrlCallback | None = None,
@@ -230,6 +231,8 @@ def upload_files(
         workunit -- either an existing ``workunit_id`` or a ``container_id``/``application_id`` to
         create one under (see :class:`UploadFilesParams`).
     :param on_progress: optional ``(filename, bytes_done, total)`` per-chunk progress callback.
+    :param on_hash_progress: optional ``(filename, bytes_done, total)`` callback fired while each file's MD5 is
+        computed, before anything is created or transferred.
     :param on_start: optional ``(total_files, total_bytes)`` callback fired once after dedup, just
         before the first transfer (never fired when everything is skipped as a duplicate). It reports
         the post-dedup file set, which is decided before ``create-resources`` runs: should the server
@@ -272,7 +275,7 @@ def upload_files(
     require_tus()
     rest = UploadRestClient(client)
     check_upload_scope(client)
-    file_infos, policies = _collect_entries(params, exclude_names)
+    file_infos, policies = _collect_entries(params, exclude_names, on_hash_progress)
 
     # Resolve the target container up front: it feeds both the duplicate check and the tus sink
     # metadata. On the reuse path it comes from the existing workunit, not from params.
@@ -475,7 +478,9 @@ def _has_resumable(
 
 
 def _collect_entries(
-    params: UploadFilesParams, exclude_names: Collection[str] | None
+    params: UploadFilesParams,
+    exclude_names: Collection[str] | None,
+    on_hash_progress: FileProgressCallback | None = None,
 ) -> tuple[list[FileInfo], dict[str, OnDuplicate]]:
     """Every file to consider, in input order, plus the ``on_duplicate`` of the entry each came from.
 
@@ -485,7 +490,9 @@ def _collect_entries(
     file_infos: list[FileInfo] = []
     policies: dict[str, OnDuplicate] = {}
     for entry in params.files:
-        for file_info in collect_file_infos([entry.path], exclude_names=exclude_names):
+        for file_info in collect_file_infos(
+            [entry.path], exclude_names=exclude_names, on_hash_progress=on_hash_progress
+        ):
             file_infos.append(file_info)
             policies[file_info.name] = entry.on_duplicate
     _reject_duplicate_names(file_infos)
diff --git a/bfabric/src/bfabric/transfer/_generic/checksums.py b/bfabric/src/bfabric/transfer/_generic/checksums.py
index 4eda5f236..89ba9c915 100644
--- a/bfabric/src/bfabric/transfer/_generic/checksums.py
+++ b/bfabric/src/bfabric/transfer/_generic/checksums.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import hashlib
+from collections.abc import Callable
 from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
@@ -8,11 +9,28 @@
     from collections.abc import Collection
     from pathlib import Path
 
+_HASH_CHUNK_SIZE = 8 * 1024 * 1024
 
-def md5_checksum(path: Path) -> str:
-    """Returns the lowercase hex MD5 digest of the file at ``path``."""
+HashProgressCallback = Callable[[str, int, int], None]
+"""Called with (filename, bytes_done, total) while a file is hashed (absolute ``bytes_done``)."""
+
+
+def md5_checksum(path: Path, on_progress: Callable[[int, int], None] | None = None) -> str:
+    """Returns the lowercase hex MD5 digest of the file at ``path``.
+
+    :param on_progress: optional ``(bytes_done, total)`` callback, called after every chunk read
+    """
     with path.open("rb") as f:
-        return hashlib.file_digest(f, "md5").hexdigest()
+        if on_progress is None:
+            return hashlib.file_digest(f, "md5").hexdigest()
+        digest = hashlib.md5()  # noqa: S324
+        total = path.stat().st_size
+        done = 0
+        while chunk := f.read(_HASH_CHUNK_SIZE):
+            digest.update(chunk)
+            done += len(chunk)
+            on_progress(done, total)
+        return digest.hexdigest()
 
 
 @dataclass
@@ -41,7 +59,12 @@ def resolve_paths(paths: list[Path]) -> list[Path]:
     return result
 
 
-def collect_file_infos(paths: list[Path], *, exclude_names: Collection[str] | None = None) -> list[FileInfo]:
+def collect_file_infos(
+    paths: list[Path],
+    *,
+    exclude_names: Collection[str] | None = None,
+    on_hash_progress: HashProgressCallback | None = None,
+) -> list[FileInfo]:
     """Expand any directories and compute a FileInfo for every resulting file.
 
     Directories are expanded recursively, preserving the path relative to the
@@ -51,6 +74,8 @@ def collect_file_infos(paths: list[Path], *, exclude_names: Collection[str] | No
     ``exclude_names`` drops files by *basename* at any depth (e.g. a sentinel or ``.DS_Store``).
     Excluding is done here rather than by the caller pre-filtering, because passing a flat file list
     loses the ``base_dir`` that gives nested files their relative resource name.
+
+    ``on_hash_progress`` receives ``(name, bytes_done, total)`` while each file is hashed.
     """
     excluded = frozenset(exclude_names or ())
     infos: list[FileInfo] = []
@@ -60,13 +85,15 @@ def collect_file_infos(paths: list[Path], *, exclude_names: Collection[str] | No
             if not expanded:
                 raise ValueError(f"Directory '{p}' contains no files.")
             for ep in expanded:
-                infos.append(compute_file_info(ep, base_dir=p))
+                infos.append(compute_file_info(ep, base_dir=p, on_hash_progress=on_hash_progress))
         elif p.name not in excluded:
-            infos.append(compute_file_info(p))
+            infos.append(compute_file_info(p, on_hash_progress=on_hash_progress))
     return infos
 
 
-def compute_file_info(path: Path, base_dir: Path | None = None) -> FileInfo:
+def compute_file_info(
+    path: Path, base_dir: Path | None = None, on_hash_progress: HashProgressCallback | None = None
+) -> FileInfo:
     """Compute MD5 checksum and size for a file.
 
     When base_dir is provided, the file name is set to the path relative to base_dir
@@ -75,7 +102,7 @@ def compute_file_info(path: Path, base_dir: Path | None = None) -> FileInfo:
     name = str(path.relative_to(base_dir)) if base_dir is not None else path.name
     return FileInfo(
         name=name,
-        md5=md5_checksum(path),
+        md5=md5_checksum(path, (lambda done, total: on_hash_progress(name, done, total)) if on_hash_progress else None),
         size=path.stat().st_size,
         path=path,
     )
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/login/oauth_login.py b/bfabric_scripts/src/bfabric_scripts/cli/login/oauth_login.py
index 58b5e9425..c32358f29 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/login/oauth_login.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/login/oauth_login.py
@@ -162,7 +162,6 @@ def cmd_auth_login(
         return
 
     print(f"Requesting scope: {params.scope}", file=sys.stderr)
-    print("Opening browser for authentication...", file=sys.stderr)
     print("Waiting for login to complete...", file=sys.stderr)
     try:
         token = pkce_login(
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py b/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py
index ba3c79231..18983aeda 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py
@@ -91,6 +91,7 @@ def cmd_workunit_upload(params: UploadParams, *, client: Bfabric) -> None:
                 track_job=params.track_job,
             ),
             on_progress=reporter.on_progress if reporter else None,
+            on_hash_progress=reporter.on_hash_progress if reporter else None,
             on_start=reporter.on_start if reporter else None,
             on_file_done=reporter.on_file_done if reporter else None,
         )
@@ -140,6 +141,16 @@ def on_start(self, total_files: int, _total_bytes: int) -> None:
         self._total_files = total_files
         self._overall_task = self._overall.add_task("Overall", total=total_files)
 
+    def on_hash_progress(self, filename: str, bytes_done: int, total: int) -> None:
+        key = f"hash:{filename}"
+        task_id = self._file_tasks.get(key)
+        if task_id is None:
+            task_id = self._current.add_task(f"Hashing {filename}", total=total)
+            self._file_tasks[key] = task_id
+        self._current.update(task_id, completed=bytes_done, total=total)
+        if bytes_done >= total:
+            self._current.remove_task(self._file_tasks.pop(key))
+
     def on_progress(self, filename: str, bytes_done: int, total: int) -> None:
         task_id = self._file_tasks.get(filename)
         if task_id is None:
diff --git a/tests/bfabric/oauth/test_pkce.py b/tests/bfabric/oauth/test_pkce.py
index e043b44ae..ba5a19d24 100644
--- a/tests/bfabric/oauth/test_pkce.py
+++ b/tests/bfabric/oauth/test_pkce.py
@@ -3,6 +3,7 @@
 import hashlib
 import base64
 import threading
+import webbrowser
 
 import httpx
 import pytest
@@ -13,6 +14,7 @@
     _CallbackServer,
     _generate_challenge,
     _generate_verifier,
+    _graphical_browser_available,
     exchange_code,
     pkce_login,
 )
@@ -215,7 +217,46 @@ def test_raises_on_http_error(self, mocker, mock_post):
             self.exchange()
 
 
+class TestGraphicalBrowserAvailable:
+    @pytest.fixture(autouse=True)
+    def clean_env(self, monkeypatch):
+        for name in ("SSH_CONNECTION", "SSH_TTY", "DISPLAY", "WAYLAND_DISPLAY"):
+            monkeypatch.delenv(name, raising=False)
+        monkeypatch.setattr("bfabric.oauth._pkce.sys.platform", "linux")
+
+    def test_false_over_ssh(self, monkeypatch):
+        monkeypatch.setenv("DISPLAY", ":0")
+        monkeypatch.setenv("SSH_CONNECTION", "1.2.3.4 1 5.6.7.8 22")
+        assert _graphical_browser_available() is False
+
+    def test_false_without_display_on_linux(self):
+        assert _graphical_browser_available() is False
+
+    def test_false_for_text_browser(self, mocker, monkeypatch):
+        monkeypatch.setenv("DISPLAY", ":0")
+        browser = mocker.MagicMock()
+        browser.name = "w3m"
+        mocker.patch("bfabric.oauth._pkce.webbrowser.get", return_value=browser)
+        assert _graphical_browser_available() is False
+
+    def test_true_with_display_and_gui_browser(self, mocker, monkeypatch):
+        monkeypatch.setenv("DISPLAY", ":0")
+        browser = mocker.MagicMock()
+        browser.name = "firefox"
+        mocker.patch("bfabric.oauth._pkce.webbrowser.get", return_value=browser)
+        assert _graphical_browser_available() is True
+
+    def test_false_when_no_browser_registered(self, mocker, monkeypatch):
+        monkeypatch.setenv("DISPLAY", ":0")
+        mocker.patch("bfabric.oauth._pkce.webbrowser.get", side_effect=webbrowser.Error)
+        assert _graphical_browser_available() is False
+
+
 class TestPkceLogin:
+    @pytest.fixture(autouse=True)
+    def graphical_browser(self, mocker):
+        mocker.patch("bfabric.oauth._pkce._graphical_browser_available", return_value=True)
+
     def test_happy_path(self, mocker):
         token_dict = {"access_token": "jwt_here", "refresh_token": "rt_here"}
 
diff --git a/tests/bfabric/transfer/generic/test_checksums.py b/tests/bfabric/transfer/generic/test_checksums.py
index 705727289..fc4cd07a4 100644
--- a/tests/bfabric/transfer/generic/test_checksums.py
+++ b/tests/bfabric/transfer/generic/test_checksums.py
@@ -20,6 +20,28 @@ def test_md5_checksum(tmp_path: Path) -> None:
     assert md5_checksum(f) == "03d01978f780dc8c34c0e279df48a8ce"
 
 
+class TestHashProgress:
+    def test_md5_reports_progress_and_same_digest(self, tmp_path: Path, mocker) -> None:
+        mocker.patch("bfabric.transfer._generic.checksums._HASH_CHUNK_SIZE", 4)
+        f = tmp_path / "data.bin"
+        f.write_bytes(b"some bytes to hash")
+        calls: list[tuple[int, int]] = []
+
+        digest = md5_checksum(f, lambda done, total: calls.append((done, total)))
+
+        assert digest == "03d01978f780dc8c34c0e279df48a8ce"
+        assert calls == [(4, 18), (8, 18), (12, 18), (16, 18), (18, 18)]
+
+    def test_collect_file_infos_reports_relative_name(self, tmp_path: Path) -> None:
+        (tmp_path / "sub").mkdir()
+        (tmp_path / "sub" / "a.txt").write_bytes(b"abc")
+        calls: list[tuple[str, int, int]] = []
+
+        collect_file_infos([tmp_path], on_hash_progress=lambda *args: calls.append(args))
+
+        assert calls == [("sub/a.txt", 3, 3)]
+
+
 def test_compute_file_info_basename(tmp_path: Path) -> None:
     f = tmp_path / "hello.txt"
     f.write_bytes(b"hello world")

From db97e3f952b10ba4215e15dbf4025a0a44aedf65 Mon Sep 17 00:00:00 2001
From: Claudio Cannizzaro <claudio.cannizzaro@fgcz.uzh.ch>
Date: Tue, 29 Sep 2026 11:50:16 +0200
Subject: [PATCH 02/12] fix: keep import resource id in resume cache; login
 falls back to device code without a GUI browser

---
 bfabric/docs/changelog.md                     |  6 ++-
 bfabric/src/bfabric/oauth/__init__.py         |  3 +-
 bfabric/src/bfabric/oauth/_pkce.py            |  4 +-
 .../src/bfabric/operations/workunit/upload.py | 17 +++++--
 bfabric/src/bfabric/transfer/resume_cache.py  |  7 +++
 bfabric_scripts/docs/changelog.md             |  4 ++
 ...ric_executable_submitter_functionalTest.py |  2 +-
 ...bfabric_executable_submitter_gridengine.py |  2 +-
 .../bfabric_scripts/cli/login/oauth_login.py  | 31 ++++++++----
 tests/bfabric/oauth/test_pkce.py              | 14 +++---
 .../operations/workunit/test_upload_files.py  | 49 +++++++++++++++++++
 tests/bfabric/transfer/test_resume_cache.py   | 19 ++++++-
 .../cli/login/test_cmd_auth_login.py          | 32 ++++++++++++
 13 files changed, 164 insertions(+), 26 deletions(-)

diff --git a/bfabric/docs/changelog.md b/bfabric/docs/changelog.md
index 7be583e24..2485d57cb 100644
--- a/bfabric/docs/changelog.md
+++ b/bfabric/docs/changelog.md
@@ -15,7 +15,11 @@ Minor breaking changes are still possible in `1.X.Y` but we try to announce them
 
 ### Changed
 
-- `pkce_login` no longer tries to open a browser over SSH, without a display, or when only a text browser is available; it prints the login URL instead.
+- `pkce_login` no longer tries to open a browser over SSH, without a display, or when only a text browser is available; it prints the login URL instead. `bfabric.oauth.graphical_browser_available` exposes that check.
+
+### Fixed
+
+- `upload_files` resuming an interrupted upload no longer fails with `importResourceIds count must match resourceIds count`. Interrupted uploads saved by an earlier version restart from byte 0 instead of resuming.
 
 ## \[1.23.0\] - 2026-09-16
 
diff --git a/bfabric/src/bfabric/oauth/__init__.py b/bfabric/src/bfabric/oauth/__init__.py
index f4da4c57b..c79b1ae3d 100644
--- a/bfabric/src/bfabric/oauth/__init__.py
+++ b/bfabric/src/bfabric/oauth/__init__.py
@@ -4,7 +4,7 @@
 from bfabric.oauth._credential_provider import OAuthCredentialProvider
 from bfabric.oauth._device_code import device_code_login
 from bfabric.oauth._endpoints import token_url
-from bfabric.oauth._pkce import AuthorizationRequest, exchange_code, pkce_login
+from bfabric.oauth._pkce import AuthorizationRequest, exchange_code, graphical_browser_available, pkce_login
 from bfabric.oauth._registration import (
     delete_client,
     read_client,
@@ -26,6 +26,7 @@
     "delete_client",
     "device_code_login",
     "exchange_code",
+    "graphical_browser_available",
     "pkce_login",
     "read_client",
     "register_client",
diff --git a/bfabric/src/bfabric/oauth/_pkce.py b/bfabric/src/bfabric/oauth/_pkce.py
index 305fcbf71..fff27a973 100644
--- a/bfabric/src/bfabric/oauth/_pkce.py
+++ b/bfabric/src/bfabric/oauth/_pkce.py
@@ -232,7 +232,7 @@ def exchange_code(
 _TEXT_BROWSERS = frozenset({"www-browser", "links", "links2", "elinks", "lynx", "w3m", "browsh"})
 
 
-def _graphical_browser_available() -> bool:
+def graphical_browser_available() -> bool:
     """Whether opening a browser here would plausibly reach a GUI the user can see.
 
     ``webbrowser.open`` reports success for a text browser or a remote session, which only strands the user.
@@ -277,7 +277,7 @@ def pkce_login(
     )
 
     browser_opened = False
-    if open_browser and _graphical_browser_available():
+    if open_browser and graphical_browser_available():
         browser_opened = webbrowser.open(request.url)
     if not browser_opened:
         print(
diff --git a/bfabric/src/bfabric/operations/workunit/upload.py b/bfabric/src/bfabric/operations/workunit/upload.py
index 7ca67c5e3..cba82c1e4 100644
--- a/bfabric/src/bfabric/operations/workunit/upload.py
+++ b/bfabric/src/bfabric/operations/workunit/upload.py
@@ -428,8 +428,13 @@ def _adopted_uploads(
             container_id=container_id,
             application_id=params.application_id,
         )
-        if entry is not None:
-            found[file_info.name] = entry
+        if entry is None:
+            continue
+        if entry.import_resource_id is None:
+            # Saved before the id was recorded: the token request cannot be rebuilt, so it cannot resume.
+            logger.info("Saved upload of {} predates resume support for tokens; uploading afresh.", file_info.name)
+            continue
+        found[file_info.name] = entry
     if not found:
         return {}
     workunit_id = next(iter(found.values())).workunit_id
@@ -455,7 +460,12 @@ def _adopted_resources(adopted: Mapping[str, ResumeEntry], to_upload: list[FileI
     needed downstream, and the saved tus URL already carries the rest.
     """
     return {
-        fi.name: CreatedResource(id=entry.resource_id, name=fi.name, storagePath=entry.storage_path)
+        fi.name: CreatedResource(
+            id=entry.resource_id,
+            name=fi.name,
+            storagePath=entry.storage_path,
+            importResourceId=entry.import_resource_id,
+        )
         for fi in to_upload
         if (entry := adopted.get(fi.name)) is not None
     }
@@ -885,6 +895,7 @@ def _report(url: str) -> None:
             application_id=application_id,
             storage_path=resource.storage_path,
             job_id=job_id,
+            import_resource_id=resource.import_resource_id,
         )
         if forward is not None:
             forward(url)
diff --git a/bfabric/src/bfabric/transfer/resume_cache.py b/bfabric/src/bfabric/transfer/resume_cache.py
index 4f8b194f8..c2d8964a6 100644
--- a/bfabric/src/bfabric/transfer/resume_cache.py
+++ b/bfabric/src/bfabric/transfer/resume_cache.py
@@ -89,6 +89,8 @@ class ResumeEntry:
     Reused rather than recreated on adoption: the hooks key status off ``jobId``, and the URL's copy
     of it cannot be repointed.
     """
+    import_resource_id: int | None = None
+    """The import record ``create-resources`` made for the resource; the token request needs it again on resume."""
     stored_at: float = 0.0
 
 
@@ -162,6 +164,7 @@ def store(
         application_id: int | None = None,
         storage_path: str | None = None,
         job_id: int | None = None,
+        import_resource_id: int | None = None,
     ) -> None:
         """Save the resume point for ``md5``, pruning entries past the TTL.
 
@@ -178,6 +181,7 @@ def store(
             application_id=application_id,
             storage_path=storage_path,
             job_id=job_id,
+            import_resource_id=import_resource_id,
             stored_at=self._now(),
         )
         self._write(entries)
@@ -225,6 +229,7 @@ def _load(self) -> dict[str, ResumeEntry]:
             application_id = fields.get("application_id")
             storage_path = fields.get("storage_path")
             job_id = fields.get("job_id")
+            import_resource_id = fields.get("import_resource_id")
             if (
                 isinstance(url, str)
                 and isinstance(stored_at, int | float)
@@ -241,6 +246,7 @@ def _load(self) -> dict[str, ResumeEntry]:
                     application_id=application_id if isinstance(application_id, int) else None,
                     storage_path=storage_path if isinstance(storage_path, str) else None,
                     job_id=job_id if isinstance(job_id, int) else None,
+                    import_resource_id=import_resource_id if isinstance(import_resource_id, int) else None,
                     stored_at=float(stored_at),
                 )
         return parsed
@@ -259,6 +265,7 @@ def _write(self, entries: dict[str, ResumeEntry]) -> None:
                     "application_id": entry.application_id,
                     "storage_path": entry.storage_path,
                     "job_id": entry.job_id,
+                    "import_resource_id": entry.import_resource_id,
                     "stored_at": entry.stored_at,
                 }
                 for md5, entry in entries.items()
diff --git a/bfabric_scripts/docs/changelog.md b/bfabric_scripts/docs/changelog.md
index a104640de..9396ac941 100644
--- a/bfabric_scripts/docs/changelog.md
+++ b/bfabric_scripts/docs/changelog.md
@@ -1,5 +1,9 @@
 ## \[Unreleased\]
 
+### Changed
+
+- `bfabric-cli login` switches to the device-code flow over SSH, without a display, or with only a text browser, unless `--no-browser` is given. The workunit upload command shows a progress bar while hashing files.
+
 ## \[1.18.0\] - 2026-09-16
 
 ### Added
diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_executable_submitter_functionalTest.py b/bfabric_scripts/src/bfabric_scripts/bfabric_executable_submitter_functionalTest.py
index 3dd07fa90..0c51f6bea 100755
--- a/bfabric_scripts/src/bfabric_scripts/bfabric_executable_submitter_functionalTest.py
+++ b/bfabric_scripts/src/bfabric_scripts/bfabric_executable_submitter_functionalTest.py
@@ -48,7 +48,7 @@ def main() -> None:
         help="external job id is required.",
     )
 
-    (options, args) = parser.parse_args()
+    options, args = parser.parse_args()
 
     if not options.externaljobid:
         parser.error("option '-j' is required.")
diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_executable_submitter_gridengine.py b/bfabric_scripts/src/bfabric_scripts/bfabric_executable_submitter_gridengine.py
index abd65f066..c836d94c0 100755
--- a/bfabric_scripts/src/bfabric_scripts/bfabric_executable_submitter_gridengine.py
+++ b/bfabric_scripts/src/bfabric_scripts/bfabric_executable_submitter_gridengine.py
@@ -49,7 +49,7 @@ def main() -> None:
         help="external job id is required.",
     )
 
-    (options, args) = parser.parse_args()
+    options, args = parser.parse_args()
 
     if not options.externaljobid:
         parser.error("option '-j' is required.")
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/login/oauth_login.py b/bfabric_scripts/src/bfabric_scripts/cli/login/oauth_login.py
index c32358f29..f7ec20867 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/login/oauth_login.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/login/oauth_login.py
@@ -12,7 +12,7 @@
 
 import cyclopts
 
-from bfabric.oauth import OAuthCredentialProvider, device_code_login, pkce_login
+from bfabric.oauth import OAuthCredentialProvider, device_code_login, graphical_browser_available, pkce_login
 from bfabric.config import DEFAULT_CONFIG_FILE
 from bfabric.config.config_writer import write_environment_to_config
 from bfabric_scripts.cli.interactive import confirm, is_interactive
@@ -36,6 +36,7 @@
 _SET_DEFAULT_HELP = "Set this environment as the default in the config file (prompted for a new environment)."
 _BASE_URL_HELP = "B-Fabric instance URL. Omit to reuse the environment's recorded URL."
 _NO_BROWSER_HELP = "Print the authorization URL instead of opening a browser."
+_DEVICE_CODE_TIMEOUT = 600.0
 _CLIENT_ID_HELP = "OAuth client ID. Omit to reuse the environment's recorded ID, or the default 'CLI'."
 
 
@@ -139,6 +140,15 @@ def _persist(token: dict[str, object], params: _LoginParams, config_file: Path)
     print(f"Config saved to environment '{params.config_env}' in {config_file}")
 
 
+def _login_with_device_code(params: _LoginParams, config_file: Path, timeout: float) -> None:
+    try:
+        token = device_code_login(params.base_url, client_id=params.client_id, scope=params.scope, timeout=timeout)
+    except RuntimeError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        raise SystemExit(1) from None
+    _persist(token, params, config_file)
+
+
 def cmd_auth_login(
     base_url: Annotated[str | None, cyclopts.Parameter(help=_BASE_URL_HELP)] = None,
     *,
@@ -155,13 +165,21 @@ def cmd_auth_login(
 
     Run with no arguments to renew an expired login: the instance URL and scope are read back from
     the environment. The browser must be on this machine, because the login is completed through a
-    redirect to a local port — over SSH, use ``auth device-code`` instead.
+    redirect to a local port. Over SSH, without a display, or with only a text browser, the device-code
+    flow (``auth device-code``) is used instead, unless ``--no-browser`` is given.
     """
     params = _resolve_params(base_url, client_id, config_env, scope, set_default, config_file)
     if params is None:
         return
 
     print(f"Requesting scope: {params.scope}", file=sys.stderr)
+    if not no_browser and not graphical_browser_available():
+        print(
+            "No graphical browser on this machine (remote session or text browser); using the device-code flow.",
+            file=sys.stderr,
+        )
+        _login_with_device_code(params, config_file, _DEVICE_CODE_TIMEOUT)
+        return
     print("Waiting for login to complete...", file=sys.stderr)
     try:
         token = pkce_login(
@@ -185,7 +203,7 @@ def cmd_auth_device_code(
     config_env: Annotated[str | None, cyclopts.Parameter(help=_CONFIG_ENV_HELP)] = None,
     config_file: Annotated[Path, cyclopts.Parameter(help="Path to the config file.")] = DEFAULT_CONFIG_FILE,
     scope: Annotated[str | None, cyclopts.Parameter(help=_SCOPE_HELP)] = None,
-    timeout: Annotated[float, cyclopts.Parameter(help="Seconds to wait for authorization.")] = 600.0,
+    timeout: Annotated[float, cyclopts.Parameter(help="Seconds to wait for authorization.")] = _DEVICE_CODE_TIMEOUT,
     set_default: Annotated[bool | None, cyclopts.Parameter(help=_SET_DEFAULT_HELP)] = None,
 ) -> None:
     """Authenticate via device code flow, for headless and remote environments.
@@ -198,9 +216,4 @@ def cmd_auth_device_code(
         return
 
     print(f"Requesting scope: {params.scope}", file=sys.stderr)
-    try:
-        token = device_code_login(params.base_url, client_id=params.client_id, scope=params.scope, timeout=timeout)
-    except RuntimeError as e:
-        print(f"Error: {e}", file=sys.stderr)
-        raise SystemExit(1) from None
-    _persist(token, params, config_file)
+    _login_with_device_code(params, config_file, timeout)
diff --git a/tests/bfabric/oauth/test_pkce.py b/tests/bfabric/oauth/test_pkce.py
index ba5a19d24..2767f48de 100644
--- a/tests/bfabric/oauth/test_pkce.py
+++ b/tests/bfabric/oauth/test_pkce.py
@@ -14,7 +14,7 @@
     _CallbackServer,
     _generate_challenge,
     _generate_verifier,
-    _graphical_browser_available,
+    graphical_browser_available,
     exchange_code,
     pkce_login,
 )
@@ -227,35 +227,35 @@ def clean_env(self, monkeypatch):
     def test_false_over_ssh(self, monkeypatch):
         monkeypatch.setenv("DISPLAY", ":0")
         monkeypatch.setenv("SSH_CONNECTION", "1.2.3.4 1 5.6.7.8 22")
-        assert _graphical_browser_available() is False
+        assert graphical_browser_available() is False
 
     def test_false_without_display_on_linux(self):
-        assert _graphical_browser_available() is False
+        assert graphical_browser_available() is False
 
     def test_false_for_text_browser(self, mocker, monkeypatch):
         monkeypatch.setenv("DISPLAY", ":0")
         browser = mocker.MagicMock()
         browser.name = "w3m"
         mocker.patch("bfabric.oauth._pkce.webbrowser.get", return_value=browser)
-        assert _graphical_browser_available() is False
+        assert graphical_browser_available() is False
 
     def test_true_with_display_and_gui_browser(self, mocker, monkeypatch):
         monkeypatch.setenv("DISPLAY", ":0")
         browser = mocker.MagicMock()
         browser.name = "firefox"
         mocker.patch("bfabric.oauth._pkce.webbrowser.get", return_value=browser)
-        assert _graphical_browser_available() is True
+        assert graphical_browser_available() is True
 
     def test_false_when_no_browser_registered(self, mocker, monkeypatch):
         monkeypatch.setenv("DISPLAY", ":0")
         mocker.patch("bfabric.oauth._pkce.webbrowser.get", side_effect=webbrowser.Error)
-        assert _graphical_browser_available() is False
+        assert graphical_browser_available() is False
 
 
 class TestPkceLogin:
     @pytest.fixture(autouse=True)
     def graphical_browser(self, mocker):
-        mocker.patch("bfabric.oauth._pkce._graphical_browser_available", return_value=True)
+        mocker.patch("bfabric.oauth._pkce.graphical_browser_available", return_value=True)
 
     def test_happy_path(self, mocker):
         token_dict = {"access_token": "jwt_here", "refresh_token": "rt_here"}
diff --git a/tests/bfabric/operations/workunit/test_upload_files.py b/tests/bfabric/operations/workunit/test_upload_files.py
index 40d594f85..edf885afa 100644
--- a/tests/bfabric/operations/workunit/test_upload_files.py
+++ b/tests/bfabric/operations/workunit/test_upload_files.py
@@ -8,6 +8,8 @@
 
 from __future__ import annotations
 
+import json
+
 from pathlib import Path
 
 import pytest
@@ -1041,6 +1043,50 @@ def _report_then_fail(*a, **kw):
 
         assert mock_send.call_args.kwargs.get("resume_url") == "https://tus/abc"
 
+    def test_resumed_run_requests_a_token_with_the_original_import_resource_ids(
+        self, tmp_path, mock_client, rest, mock_send
+    ):
+        """The token request needs one import id per resource id; a resumed file has no
+        ``create-resources`` record to take it from, so the cache must carry it."""
+        self._setup(rest)
+        cache = tmp_path / "resume.json"
+
+        def _report_then_fail(*a, **kw):
+            kw["on_url"]("https://tus/abc")
+            raise TransferError("connection reset")
+
+        mock_send.side_effect = _report_then_fail
+        _ = upload_files(mock_client, _params("/src/a.txt"), resume_cache=cache)
+
+        mock_send.side_effect = None
+        rest.get_upload_token.reset_mock()
+        _ = upload_files(mock_client, _params("/src/a.txt"), resume_cache=cache)
+
+        assert rest.get_upload_token.call_args.args[1:3] == ([10], [90])
+
+    def test_entry_without_import_resource_id_is_not_resumed(self, tmp_path, mock_client, rest, mock_send):
+        """Entries saved before the id was recorded cannot mint a valid token; upload afresh."""
+        self._setup(rest)
+        cache = tmp_path / "resume.json"
+
+        def _report_then_fail(*a, **kw):
+            kw["on_url"]("https://tus/abc")
+            raise TransferError("connection reset")
+
+        mock_send.side_effect = _report_then_fail
+        _ = upload_files(mock_client, _params("/src/a.txt"), resume_cache=cache)
+        document = json.loads(cache.read_text())
+        for entry in document["entries"].values():
+            del entry["import_resource_id"]
+        cache.write_text(json.dumps(document))
+
+        mock_send.side_effect = None
+        rest.create_resources.reset_mock()
+        _ = upload_files(mock_client, _params("/src/a.txt"), resume_cache=cache)
+
+        rest.create_resources.assert_called_once()
+        assert mock_send.call_args.kwargs.get("resume_url") is None
+
     def test_entry_is_discarded_once_the_file_transfers(self, tmp_path, mock_client, rest, mock_send):
         self._setup(rest)
         cache = tmp_path / "resume.json"
@@ -1062,6 +1108,7 @@ def test_stale_url_falls_back_to_a_fresh_upload(self, tmp_path, mock_client, res
             workunit_id=WORKUNIT_ID,
             resource_id=10,
             container_id=100,
+            import_resource_id=90,
         )
         attempts: list[str | None] = []
 
@@ -1098,6 +1145,7 @@ def test_cross_origin_saved_url_is_ignored(self, tmp_path, mock_client, rest, mo
             workunit_id=WORKUNIT_ID,
             resource_id=10,
             container_id=100,
+            import_resource_id=90,
         )
 
         _ = upload_files(mock_client, _params("/src/a.txt"), resume_cache=cache)
@@ -1304,6 +1352,7 @@ def _store(cache: Path, *, name: str, url: str, workunit_id: int, resource_id: i
             container_id=100,
             application_id=5,
             storage_path=f"/store/{name}",
+            import_resource_id=resource_id + 80,
         )
 
     def test_two_files_with_identical_bytes_keep_their_own_resources(
diff --git a/tests/bfabric/transfer/test_resume_cache.py b/tests/bfabric/transfer/test_resume_cache.py
index 6aaae0aed..d1de78c87 100644
--- a/tests/bfabric/transfer/test_resume_cache.py
+++ b/tests/bfabric/transfer/test_resume_cache.py
@@ -19,7 +19,17 @@
 PATH = "/data/probe.raw"
 
 
-def _store(cache, *, md5, url=URL, workunit_id=900, resource_id=700, container_id=CONTAINER_ID, path=PATH):
+def _store(
+    cache,
+    *,
+    md5,
+    url=URL,
+    workunit_id=900,
+    resource_id=700,
+    container_id=CONTAINER_ID,
+    path=PATH,
+    import_resource_id=800,
+):
     """Store an entry, defaulting the records a resumed upload must continue into."""
     cache.store(
         md5=md5,
@@ -29,6 +39,7 @@ def _store(cache, *, md5, url=URL, workunit_id=900, resource_id=700, container_i
         resource_id=resource_id,
         container_id=container_id,
         application_id=APPLICATION_ID,
+        import_resource_id=import_resource_id,
     )
 
 
@@ -97,6 +108,12 @@ def test_restoring_the_same_md5_overwrites(self, cache_path, clock):
 
         assert _url(cache, md5="aaa", endpoint=ENDPOINT) == URL + "-new"
 
+    def test_import_resource_id_round_trips(self, cache_path, clock):
+        cache = ResumeCache(cache_path, now=clock)
+        _store(cache, md5="aaa", import_resource_id=812)
+        entry = _lookup(cache, md5="aaa")
+        assert entry is not None and entry.import_resource_id == 812
+
 
 class TestInvalidation:
     def test_cross_origin_entry_is_not_returned(self, cache_path, clock):
diff --git a/tests/bfabric_scripts/cli/login/test_cmd_auth_login.py b/tests/bfabric_scripts/cli/login/test_cmd_auth_login.py
index 08d7c7550..977a226a1 100644
--- a/tests/bfabric_scripts/cli/login/test_cmd_auth_login.py
+++ b/tests/bfabric_scripts/cli/login/test_cmd_auth_login.py
@@ -10,7 +10,39 @@
 from bfabric_scripts.cli.login.oauth_login import cmd_auth_login
 
 
+@pytest.fixture(autouse=True)
+def graphical_browser(mocker):
+    """Pretend a GUI browser exists, so the tests don't depend on the machine running them."""
+    mocker.patch("bfabric_scripts.cli.login.oauth_login.graphical_browser_available", return_value=True)
+
+
 class TestCmdAuthLogin:
+    def test_uses_device_code_without_graphical_browser(self, tmp_path, mocker, oauth_token, oauth_session):
+        mocker.patch("bfabric_scripts.cli.login.oauth_login.graphical_browser_available", return_value=False)
+        mock_pkce = mocker.patch("bfabric_scripts.cli.login.oauth_login.pkce_login")
+        mock_dc = mocker.patch("bfabric_scripts.cli.login.oauth_login.device_code_login", return_value=oauth_token)
+        config_file = tmp_path / "config.yml"
+        cmd_auth_login(
+            base_url="https://example.com/bfabric", scope="api:read", config_env="PROD", config_file=config_file
+        )
+        mock_dc.assert_called_once()
+        mock_pkce.assert_not_called()
+        assert yaml.safe_load(config_file.read_text())["PROD"]["auth_method"] == "oauth"
+
+    def test_no_browser_flag_keeps_pkce(self, tmp_path, mocker, oauth_token, oauth_session):
+        mocker.patch("bfabric_scripts.cli.login.oauth_login.graphical_browser_available", return_value=False)
+        mock_pkce = mocker.patch("bfabric_scripts.cli.login.oauth_login.pkce_login", return_value=oauth_token)
+        mock_dc = mocker.patch("bfabric_scripts.cli.login.oauth_login.device_code_login")
+        cmd_auth_login(
+            base_url="https://example.com/bfabric",
+            scope="api:read",
+            config_env="PROD",
+            config_file=tmp_path / "config.yml",
+            no_browser=True,
+        )
+        mock_pkce.assert_called_once()
+        mock_dc.assert_not_called()
+
     def test_writes_config_and_caches_token(self, tmp_path, mocker, oauth_token, oauth_session):
         config_file = tmp_path / "config.yml"
         mock_pkce = mocker.patch("bfabric_scripts.cli.login.oauth_login.pkce_login", return_value=oauth_token)

From 62ec1bba22f61260c89d53d31c76c33c2bac4423 Mon Sep 17 00:00:00 2001
From: Claudio Cannizzaro <claudio.cannizzaro@fgcz.uzh.ch>
Date: Tue, 29 Sep 2026 11:58:53 +0200
Subject: [PATCH 03/12] feat: cache file MD5s by path/size/mtime and hash
 upload entries in parallel

---
 bfabric/docs/changelog.md                     |   1 +
 bfabric/src/bfabric/config/config_data.py     |   1 +
 .../src/bfabric/operations/workunit/upload.py |  45 +++++-
 .../bfabric/transfer/_generic/checksums.py    |  36 +++--
 bfabric/src/bfabric/transfer/hash_cache.py    | 137 ++++++++++++++++++
 bfabric_scripts/docs/changelog.md             |   2 +-
 .../bfabric_scripts/cli/workunit/upload.py    |   9 ++
 .../operations/workunit/test_upload_files.py  |  43 ++++++
 .../transfer/generic/test_checksums.py        |  27 ++++
 tests/bfabric/transfer/test_hash_cache.py     |  73 ++++++++++
 10 files changed, 356 insertions(+), 18 deletions(-)
 create mode 100644 bfabric/src/bfabric/transfer/hash_cache.py
 create mode 100644 tests/bfabric/transfer/test_hash_cache.py

diff --git a/bfabric/docs/changelog.md b/bfabric/docs/changelog.md
index 2485d57cb..e0a0c2af6 100644
--- a/bfabric/docs/changelog.md
+++ b/bfabric/docs/changelog.md
@@ -12,6 +12,7 @@ Minor breaking changes are still possible in `1.X.Y` but we try to announce them
 ### Added
 
 - `upload_files` accepts `on_hash_progress`, called while each file's MD5 is computed; `bfabric-cli workunit upload` shows it as a progress bar.
+- `upload_files` remembers each file's MD5 by path, size and mtime (`hash_cache`, default `~/.bfabric/hashes.json`), so a retry or resumed upload skips re-hashing unchanged files. `hash_workers` hashes several entries concurrently.
 
 ### Changed
 
diff --git a/bfabric/src/bfabric/config/config_data.py b/bfabric/src/bfabric/config/config_data.py
index 6ac011237..fb49f631b 100644
--- a/bfabric/src/bfabric/config/config_data.py
+++ b/bfabric/src/bfabric/config/config_data.py
@@ -39,6 +39,7 @@ def _gather_auth_config(cls, values: object) -> object:
         """Accept the flat ``auth_method``/``client_id``/... kwargs the override JSON carries."""
         if not isinstance(values, dict):
             return values
+
         raw = cast("dict[str, object]", values)
         if "auth_config" in raw or not any(raw.get(key) is not None for key in _FLAT_AUTH_KEYS):
             return raw
diff --git a/bfabric/src/bfabric/operations/workunit/upload.py b/bfabric/src/bfabric/operations/workunit/upload.py
index cba82c1e4..cd2736fab 100644
--- a/bfabric/src/bfabric/operations/workunit/upload.py
+++ b/bfabric/src/bfabric/operations/workunit/upload.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from collections import Counter
+from concurrent.futures import ThreadPoolExecutor
 from collections.abc import Callable, Mapping
 from dataclasses import dataclass, field, replace
 from pathlib import Path
@@ -24,6 +25,7 @@
     tus_sink_for_resource,
 )
 from bfabric.transfer._generic.origin import same_origin
+from bfabric.transfer.hash_cache import HashCache, compute_hash_cache_path
 from bfabric.transfer.resume_cache import ResumeCache, ResumeEntry, compute_resume_cache_path
 
 if TYPE_CHECKING:
@@ -54,6 +56,9 @@
 caller must still be able to ask for.
 """
 
+_USE_DEFAULT_HASH_CACHE: Final = Path("<default>")
+"""Sentinel for ``hash_cache``, for the same reason as ``_USE_DEFAULT_RESUME_CACHE``."""
+
 OnDuplicate = Literal["upload", "skip", "link"]
 """What to do with a file whose content the target container already stores.
 
@@ -195,6 +200,8 @@ def upload_files(
     audit_attributes: dict[str, str] | None = None,
     exclude_names: Collection[str] | None = None,
     resume_cache: Path | None = _USE_DEFAULT_RESUME_CACHE,
+    hash_cache: Path | None = _USE_DEFAULT_HASH_CACHE,
+    hash_workers: int = 1,
 ) -> UploadSummary:
     """Upload files to a B-Fabric workunit over tus, end to end.
 
@@ -256,6 +263,12 @@ def upload_files(
         dropped once its file transfers, and ignored when it is stale, past its TTL, no longer
         same-origin with the tus endpoint, or was stored for a different container/application -- in
         each of those cases the file is uploaded afresh.
+    :param hash_cache: path to a JSON file remembering each file's MD5 by path, size and mtime, so a re-run
+        (a retry, or resuming an interrupted upload) does not re-read unchanged files. Left unset,
+        ``~/.bfabric/hashes.json`` is used; ``None`` keeps no state and always hashes.
+    :param hash_workers: how many entries are hashed concurrently (``1`` = sequentially). Each entry
+        (a file or a directory) is hashed by one worker, so this only helps with several entries.
+        ``on_hash_progress`` is then called from worker threads.
     :param exclude_names: basenames to skip at any depth (e.g. a sentinel file the caller drops in
         the folder, or ``.DS_Store``). Filter here rather than pre-filtering ``files`` yourself: a
         flat file list loses the directory that gives nested files their relative resource name.
@@ -275,7 +288,14 @@ def upload_files(
     require_tus()
     rest = UploadRestClient(client)
     check_upload_scope(client)
-    file_infos, policies = _collect_entries(params, exclude_names, on_hash_progress)
+    hash_cache_path = compute_hash_cache_path().expanduser() if hash_cache is _USE_DEFAULT_HASH_CACHE else hash_cache
+    hashes = HashCache(hash_cache_path) if hash_cache_path is not None else None
+    try:
+        file_infos, policies = _collect_entries(params, exclude_names, on_hash_progress, hashes, hash_workers)
+    finally:
+        # Also on interrupt: the files hashed so far are the point of keeping the cache.
+        if hashes is not None:
+            hashes.flush()
 
     # Resolve the target container up front: it feeds both the duplicate check and the tus sink
     # metadata. On the reuse path it comes from the existing workunit, not from params.
@@ -491,18 +511,31 @@ def _collect_entries(
     params: UploadFilesParams,
     exclude_names: Collection[str] | None,
     on_hash_progress: FileProgressCallback | None = None,
+    hash_cache: HashCache | None = None,
+    hash_workers: int = 1,
 ) -> tuple[list[FileInfo], dict[str, OnDuplicate]]:
     """Every file to consider, in input order, plus the ``on_duplicate`` of the entry each came from.
 
     The collector is called per entry because it flattens directories, losing which entry a file came
-    from -- and that is what carries the policy.
+    from -- and that is what carries the policy. Entries are hashed by up to ``hash_workers`` threads;
+    the result order is the input order regardless.
     """
+
+    def collect(entry: UploadFileParam) -> list[FileInfo]:
+        return collect_file_infos(
+            [entry.path], exclude_names=exclude_names, on_hash_progress=on_hash_progress, hash_cache=hash_cache
+        )
+
+    if hash_workers > 1 and len(params.files) > 1:
+        with ThreadPoolExecutor(max_workers=min(hash_workers, len(params.files))) as pool:
+            collected = list(pool.map(collect, params.files))
+    else:
+        collected = [collect(entry) for entry in params.files]
+
     file_infos: list[FileInfo] = []
     policies: dict[str, OnDuplicate] = {}
-    for entry in params.files:
-        for file_info in collect_file_infos(
-            [entry.path], exclude_names=exclude_names, on_hash_progress=on_hash_progress
-        ):
+    for entry, infos in zip(params.files, collected, strict=True):
+        for file_info in infos:
             file_infos.append(file_info)
             policies[file_info.name] = entry.on_duplicate
     _reject_duplicate_names(file_infos)
diff --git a/bfabric/src/bfabric/transfer/_generic/checksums.py b/bfabric/src/bfabric/transfer/_generic/checksums.py
index 89ba9c915..bbc83457f 100644
--- a/bfabric/src/bfabric/transfer/_generic/checksums.py
+++ b/bfabric/src/bfabric/transfer/_generic/checksums.py
@@ -9,6 +9,8 @@
     from collections.abc import Collection
     from pathlib import Path
 
+    from bfabric.transfer.hash_cache import HashCache
+
 _HASH_CHUNK_SIZE = 8 * 1024 * 1024
 
 HashProgressCallback = Callable[[str, int, int], None]
@@ -64,6 +66,7 @@ def collect_file_infos(
     *,
     exclude_names: Collection[str] | None = None,
     on_hash_progress: HashProgressCallback | None = None,
+    hash_cache: HashCache | None = None,
 ) -> list[FileInfo]:
     """Expand any directories and compute a FileInfo for every resulting file.
 
@@ -75,7 +78,9 @@ def collect_file_infos(
     Excluding is done here rather than by the caller pre-filtering, because passing a flat file list
     loses the ``base_dir`` that gives nested files their relative resource name.
 
-    ``on_hash_progress`` receives ``(name, bytes_done, total)`` while each file is hashed.
+    ``on_hash_progress`` receives ``(name, bytes_done, total)`` while each file is hashed. ``hash_cache``
+    supplies the MD5 of a file whose size and mtime are unchanged, skipping the read; new hashes are
+    added to it but not flushed.
     """
     excluded = frozenset(exclude_names or ())
     infos: list[FileInfo] = []
@@ -85,24 +90,33 @@ def collect_file_infos(
             if not expanded:
                 raise ValueError(f"Directory '{p}' contains no files.")
             for ep in expanded:
-                infos.append(compute_file_info(ep, base_dir=p, on_hash_progress=on_hash_progress))
+                infos.append(
+                    compute_file_info(ep, base_dir=p, on_hash_progress=on_hash_progress, hash_cache=hash_cache)
+                )
         elif p.name not in excluded:
-            infos.append(compute_file_info(p, on_hash_progress=on_hash_progress))
+            infos.append(compute_file_info(p, on_hash_progress=on_hash_progress, hash_cache=hash_cache))
     return infos
 
 
 def compute_file_info(
-    path: Path, base_dir: Path | None = None, on_hash_progress: HashProgressCallback | None = None
+    path: Path,
+    base_dir: Path | None = None,
+    on_hash_progress: HashProgressCallback | None = None,
+    hash_cache: HashCache | None = None,
 ) -> FileInfo:
     """Compute MD5 checksum and size for a file.
 
     When base_dir is provided, the file name is set to the path relative to base_dir
-    (e.g. "subdir/file.txt"). Otherwise, just the basename is used.
+    (e.g. "subdir/file.txt"). Otherwise, just the basename is used. With a ``hash_cache``, a file whose
+    size and mtime match a recorded entry is not re-read.
     """
     name = str(path.relative_to(base_dir)) if base_dir is not None else path.name
-    return FileInfo(
-        name=name,
-        md5=md5_checksum(path, (lambda done, total: on_hash_progress(name, done, total)) if on_hash_progress else None),
-        size=path.stat().st_size,
-        path=path,
-    )
+    # Stat before reading: a file that changes mid-hash then mismatches on the next lookup.
+    stat = path.stat()
+    md5 = hash_cache.get(path, size=stat.st_size, mtime_ns=stat.st_mtime_ns) if hash_cache is not None else None
+    if md5 is None:
+        progress = (lambda done, total: on_hash_progress(name, done, total)) if on_hash_progress else None
+        md5 = md5_checksum(path, progress)
+        if hash_cache is not None:
+            hash_cache.put(path, size=stat.st_size, mtime_ns=stat.st_mtime_ns, md5=md5)
+    return FileInfo(name=name, md5=md5, size=stat.st_size, path=path)
diff --git a/bfabric/src/bfabric/transfer/hash_cache.py b/bfabric/src/bfabric/transfer/hash_cache.py
new file mode 100644
index 000000000..1fdcd9809
--- /dev/null
+++ b/bfabric/src/bfabric/transfer/hash_cache.py
@@ -0,0 +1,137 @@
+"""Disk-backed store of file MD5s, so re-running an upload need not re-read multi-GB files.
+
+An entry is valid while the file's size and mtime are unchanged -- the same heuristic git and rsync
+use. A file rewritten in place with identical size and mtime would be missed, but the server verifies
+the checksum after the transfer, so the worst outcome is a resource marked failed, never silently
+wrong data.
+
+Like the resume cache this is an optimisation, never a source of truth: a miss, a corrupt file, or a
+failed write costs a re-hash and nothing more, so every failure is swallowed rather than raised.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+import time
+from pathlib import Path
+from typing import TYPE_CHECKING, final
+
+from loguru import logger
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+DEFAULT_HASH_TTL_SECONDS = 30 * 24 * 60 * 60
+"""How long an unused entry is kept; pruning stops the file from growing without bound."""
+
+_FORMAT_VERSION = 1
+
+
+def compute_hash_cache_path() -> Path:
+    """The default hash-cache path, ``~/.bfabric/hashes.json``.
+
+    Not per server, unlike the resume cache: a file's MD5 does not depend on where it is uploaded.
+    """
+    return Path("~/.bfabric/hashes.json")
+
+
+@final
+class HashCache:
+    """Maps ``(file path, size, mtime)`` to an MD5, persisted as JSON with 0o600 permissions.
+
+    Thread-safe. Lookups see the entries on disk plus those added since; :meth:`flush` writes the
+    additions once, so hashing many files does not rewrite the file per file.
+    """
+
+    def __init__(
+        self,
+        path: Path,
+        *,
+        ttl_seconds: int = DEFAULT_HASH_TTL_SECONDS,
+        now: Callable[[], float] | None = None,
+    ) -> None:
+        self._path: Path = path
+        self._ttl: int = ttl_seconds
+        self._now: Callable[[], float] = now if now is not None else time.time
+        self._lock: threading.Lock = threading.Lock()
+        self._on_disk: dict[str, dict[str, object]] | None = None
+        self._added: dict[str, dict[str, object]] = {}
+
+    def get(self, path: Path, *, size: int, mtime_ns: int) -> str | None:
+        """The MD5 recorded for ``path`` if its size and mtime still match, else ``None``."""
+        key = str(path.absolute())
+        with self._lock:
+            entry = self._added.get(key) or self._disk().get(key)
+        if entry is None or entry.get("size") != size or entry.get("mtime_ns") != mtime_ns:
+            return None
+        md5 = entry.get("md5")
+        return md5 if isinstance(md5, str) else None
+
+    def put(self, path: Path, *, size: int, mtime_ns: int, md5: str) -> None:
+        """Remember the MD5 of ``path`` in memory; :meth:`flush` persists it."""
+        with self._lock:
+            self._added[str(path.absolute())] = {
+                "size": size,
+                "mtime_ns": mtime_ns,
+                "md5": md5,
+                "stored_at": self._now(),
+            }
+
+    def flush(self) -> None:
+        """Write the entries added so far to disk, merged with what is there, pruning expired ones."""
+        with self._lock:
+            if not self._added:
+                return
+            self._on_disk = None  # re-read: another process may have written since we loaded
+            merged = {**self._disk(), **self._added}
+            cutoff = self._now() - self._ttl
+            entries = {key: entry for key, entry in merged.items() if _stored_at(entry) >= cutoff}
+            if self._write(entries):
+                self._on_disk = entries
+                self._added = {}
+
+    def _disk(self) -> dict[str, dict[str, object]]:
+        if self._on_disk is None:
+            self._on_disk = self._read()
+        return self._on_disk
+
+    def _read(self) -> dict[str, dict[str, object]]:
+        try:
+            raw: object = json.loads(self._path.read_text())  # pyright: ignore[reportAny]
+        except (OSError, json.JSONDecodeError):
+            return {}
+        if (
+            not isinstance(raw, dict) or raw.get("version") != _FORMAT_VERSION
+        ):  # pyright: ignore[reportUnknownMemberType]
+            return {}
+        entries = raw.get("entries")  # pyright: ignore[reportUnknownMemberType]
+        if not isinstance(entries, dict):
+            return {}
+        return {
+            key: entry for key, entry in entries.items() if isinstance(entry, dict)
+        }  # pyright: ignore[reportUnknownVariableType]
+
+    def _write(self, entries: dict[str, dict[str, object]]) -> bool:
+        payload = {"version": _FORMAT_VERSION, "entries": entries}
+        target = self._path
+        tmp = target.with_name(f".{target.name}.{os.getpid()}.tmp")
+        try:
+            target.parent.mkdir(parents=True, exist_ok=True)
+            fd = os.open(str(tmp), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+            try:
+                _ = os.write(fd, json.dumps(payload).encode())
+            finally:
+                os.close(fd)
+            _ = tmp.replace(target)
+        except OSError as error:  # noqa: BLE001 a cache write must never fail an upload
+            logger.warning("Could not write the hash cache at {}: {}", target, error)
+            tmp.unlink(missing_ok=True)
+            return False
+        return True
+
+
+def _stored_at(entry: dict[str, object]) -> float:
+    stored_at = entry.get("stored_at")
+    return float(stored_at) if isinstance(stored_at, int | float) else 0.0
diff --git a/bfabric_scripts/docs/changelog.md b/bfabric_scripts/docs/changelog.md
index 9396ac941..880016ed2 100644
--- a/bfabric_scripts/docs/changelog.md
+++ b/bfabric_scripts/docs/changelog.md
@@ -2,7 +2,7 @@
 
 ### Changed
 
-- `bfabric-cli login` switches to the device-code flow over SSH, without a display, or with only a text browser, unless `--no-browser` is given. The workunit upload command shows a progress bar while hashing files.
+- `bfabric-cli login` switches to the device-code flow over SSH, without a display, or with only a text browser, unless `--no-browser` is given. `bfabric-cli workunit upload` shows a progress bar while hashing, hashes 4 files at once (`--hash-workers`), and reuses checksums of unchanged files from earlier runs (`--no-hash-cache` to disable).
 
 ## \[1.18.0\] - 2026-09-16
 
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py b/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py
index 18983aeda..87ce17222 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py
@@ -24,6 +24,7 @@
 
 from bfabric import Bfabric
 from bfabric.operations.workunit import OnDuplicate, UploadFileParam, UploadFilesParams, upload_files
+from bfabric.transfer.hash_cache import compute_hash_cache_path
 from bfabric.utils.cli_integration import use_client
 
 if TYPE_CHECKING:
@@ -55,6 +56,12 @@ class UploadParams(BaseModel):
     """Show a live upload progress bar. Pass ``--no-progress`` to disable; it is also
     auto-disabled when stderr is not an interactive terminal."""
 
+    hash_workers: Annotated[int, cyclopts.Parameter(name="--hash-workers")] = 4
+    """How many files to compute checksums for at the same time (``1`` = one at a time)."""
+    hash_cache: bool = True
+    """Reuse checksums of files whose size and modification time are unchanged since a previous run
+    (``~/.bfabric/hashes.json``). Pass ``--no-hash-cache`` to always re-read every file."""
+
     @model_validator(mode="after")
     def _validate_target(self) -> UploadParams:
         if self.workunit_id is not None:
@@ -92,6 +99,8 @@ def cmd_workunit_upload(params: UploadParams, *, client: Bfabric) -> None:
             ),
             on_progress=reporter.on_progress if reporter else None,
             on_hash_progress=reporter.on_hash_progress if reporter else None,
+            hash_workers=params.hash_workers,
+            hash_cache=compute_hash_cache_path().expanduser() if params.hash_cache else None,
             on_start=reporter.on_start if reporter else None,
             on_file_done=reporter.on_file_done if reporter else None,
         )
diff --git a/tests/bfabric/operations/workunit/test_upload_files.py b/tests/bfabric/operations/workunit/test_upload_files.py
index edf885afa..6f4db86e2 100644
--- a/tests/bfabric/operations/workunit/test_upload_files.py
+++ b/tests/bfabric/operations/workunit/test_upload_files.py
@@ -26,6 +26,7 @@
 from bfabric.operations.workunit.upload import _describe_dropped
 from bfabric.transfer import CreatedResource, DuplicateResult, FileInfo, TransferError, UploadTokenResult
 from bfabric.transfer.errors import BfabricTransferError, ScopeError
+from bfabric.transfer.hash_cache import HashCache
 from bfabric.transfer.resume_cache import ResumeCache
 
 WORKUNIT_ID = 555
@@ -71,6 +72,9 @@ def isolate_resume_cache(mocker, tmp_path):
     Autouse because resuming is on by default: any test that transfers a file would otherwise write
     to the developer's home directory and leak state between runs.
     """
+    mocker.patch(
+        "bfabric.operations.workunit.upload.compute_hash_cache_path", return_value=tmp_path / "default-hashes.json"
+    )
     return mocker.patch(
         "bfabric.operations.workunit.upload.compute_resume_cache_path",
         return_value=tmp_path / "default-resume.json",
@@ -198,6 +202,45 @@ def test_records_resource_details(self, mock_client, rest, mock_send):
         assert upload.import_resource_id == 90
 
 
+class TestHashing:
+    @staticmethod
+    def _setup(rest, *names):
+        rest.create_resources.return_value = _created(*names)
+        rest.get_upload_token.return_value = UploadTokenResult(token="tok", tus_endpoint="https://tus/")
+
+    def test_parallel_hashing_keeps_input_order(self, mock_client, rest, mock_collect, mock_send):
+        names = ["a.txt", "b.txt", "c.txt", "d.txt"]
+        self._setup(rest, *names)
+
+        _ = upload_files(mock_client, _params(*(f"/src/{n}" for n in names)), hash_workers=4)
+
+        assert _created_names(rest) == names
+
+    def test_hash_cache_is_passed_to_the_collector(self, tmp_path, mock_client, rest, mock_collect, mock_send):
+        self._setup(rest, "a.txt")
+        cache_path = tmp_path / "hashes.json"
+
+        _ = upload_files(mock_client, _params("/src/a.txt"), hash_cache=cache_path)
+
+        assert isinstance(mock_collect.call_args.kwargs["hash_cache"], HashCache)
+
+    def test_no_hash_cache_passes_none(self, mock_client, rest, mock_collect, mock_send):
+        self._setup(rest, "a.txt")
+
+        _ = upload_files(mock_client, _params("/src/a.txt"), hash_cache=None)
+
+        assert mock_collect.call_args.kwargs["hash_cache"] is None
+
+    def test_cache_is_flushed_when_hashing_is_interrupted(self, tmp_path, mocker, mock_client, rest, mock_collect):
+        flush = mocker.patch("bfabric.operations.workunit.upload.HashCache.flush")
+        mock_collect.side_effect = KeyboardInterrupt
+
+        with pytest.raises(KeyboardInterrupt):
+            upload_files(mock_client, _params("/src/a.txt"), hash_cache=tmp_path / "hashes.json")
+
+        flush.assert_called_once()
+
+
 class TestExcludeNames:
     def test_forwarded_to_collector(self, mock_client, rest, mock_collect, mock_send):
         rest.create_resources.return_value = _created("src")
diff --git a/tests/bfabric/transfer/generic/test_checksums.py b/tests/bfabric/transfer/generic/test_checksums.py
index fc4cd07a4..8aba2e50d 100644
--- a/tests/bfabric/transfer/generic/test_checksums.py
+++ b/tests/bfabric/transfer/generic/test_checksums.py
@@ -42,6 +42,33 @@ def test_collect_file_infos_reports_relative_name(self, tmp_path: Path) -> None:
         assert calls == [("sub/a.txt", 3, 3)]
 
 
+class TestHashCacheUse:
+    def test_second_call_does_not_re_read_the_file(self, tmp_path: Path, mocker) -> None:
+        from bfabric.transfer.hash_cache import HashCache
+
+        f = tmp_path / "data.bin"
+        f.write_bytes(b"some bytes to hash")
+        cache = HashCache(tmp_path / "hashes.json")
+        first = compute_file_info(f, hash_cache=cache)
+
+        spy = mocker.patch("bfabric.transfer._generic.checksums.md5_checksum")
+        second = compute_file_info(f, hash_cache=cache)
+
+        spy.assert_not_called()
+        assert second == first
+
+    def test_modified_file_is_re_hashed(self, tmp_path: Path) -> None:
+        from bfabric.transfer.hash_cache import HashCache
+
+        f = tmp_path / "data.bin"
+        f.write_bytes(b"one")
+        cache = HashCache(tmp_path / "hashes.json")
+        before = compute_file_info(f, hash_cache=cache)
+        f.write_bytes(b"three")
+
+        assert compute_file_info(f, hash_cache=cache).md5 != before.md5
+
+
 def test_compute_file_info_basename(tmp_path: Path) -> None:
     f = tmp_path / "hello.txt"
     f.write_bytes(b"hello world")
diff --git a/tests/bfabric/transfer/test_hash_cache.py b/tests/bfabric/transfer/test_hash_cache.py
new file mode 100644
index 000000000..d85638b3d
--- /dev/null
+++ b/tests/bfabric/transfer/test_hash_cache.py
@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import json
+import stat
+
+import pytest
+
+from bfabric.transfer.hash_cache import HashCache
+
+
+@pytest.fixture
+def cache_path(tmp_path):
+    return tmp_path / "hashes.json"
+
+
+@pytest.fixture
+def sample(tmp_path):
+    path = tmp_path / "a.raw"
+    path.write_bytes(b"abc")
+    return path
+
+
+class TestHashCache:
+    def test_flushed_entry_is_found_by_a_new_instance(self, cache_path, sample):
+        cache = HashCache(cache_path)
+        cache.put(sample, size=3, mtime_ns=7, md5="m")
+        cache.flush()
+        assert HashCache(cache_path).get(sample, size=3, mtime_ns=7) == "m"
+
+    def test_entry_is_visible_before_flush(self, cache_path, sample):
+        cache = HashCache(cache_path)
+        cache.put(sample, size=3, mtime_ns=7, md5="m")
+        assert cache.get(sample, size=3, mtime_ns=7) == "m"
+        assert not cache_path.exists()
+
+    @pytest.mark.parametrize(("size", "mtime_ns"), [(4, 7), (3, 8)])
+    def test_changed_size_or_mtime_is_a_miss(self, cache_path, sample, size, mtime_ns):
+        cache = HashCache(cache_path)
+        cache.put(sample, size=3, mtime_ns=7, md5="m")
+        cache.flush()
+        assert HashCache(cache_path).get(sample, size=size, mtime_ns=mtime_ns) is None
+
+    def test_flush_merges_with_entries_written_by_another_instance(self, cache_path, tmp_path, sample):
+        other = tmp_path / "b.raw"
+        first, second = HashCache(cache_path), HashCache(cache_path)
+        first.put(sample, size=3, mtime_ns=1, md5="a")
+        second.put(other, size=3, mtime_ns=2, md5="b")
+        first.flush()
+        second.flush()
+        merged = HashCache(cache_path)
+        assert merged.get(sample, size=3, mtime_ns=1) == "a"
+        assert merged.get(other, size=3, mtime_ns=2) == "b"
+
+    def test_expired_entries_are_pruned_on_flush(self, cache_path, tmp_path, sample):
+        clock = [0.0]
+        cache = HashCache(cache_path, ttl_seconds=10, now=lambda: clock[0])
+        cache.put(sample, size=3, mtime_ns=1, md5="old")
+        cache.flush()
+        clock[0] = 100.0
+        cache.put(tmp_path / "b.raw", size=1, mtime_ns=1, md5="new")
+        cache.flush()
+        assert list(json.loads(cache_path.read_text())["entries"]) == [str(tmp_path / "b.raw")]
+
+    def test_file_is_private(self, cache_path, sample):
+        cache = HashCache(cache_path)
+        cache.put(sample, size=3, mtime_ns=1, md5="m")
+        cache.flush()
+        assert stat.S_IMODE(cache_path.stat().st_mode) == 0o600
+
+    @pytest.mark.parametrize("content", ["not json", "[]", '{"version": 99, "entries": {}}'])
+    def test_unreadable_file_is_a_miss_not_an_error(self, cache_path, sample, content):
+        cache_path.write_text(content)
+        assert HashCache(cache_path).get(sample, size=3, mtime_ns=1) is None

From 53170652b634f4a0598f76d5f4b5f9a952d8d894 Mon Sep 17 00:00:00 2001
From: Claudio Cannizzaro <claudio.cannizzaro@fgcz.uzh.ch>
Date: Tue, 29 Sep 2026 12:08:07 +0200
Subject: [PATCH 04/12] feat: add --chunk-size to workunit upload

---
 bfabric/docs/changelog.md                     |  1 +
 .../src/bfabric/operations/workunit/upload.py | 27 +++++++++++++++++--
 bfabric/src/bfabric/transfer/_generic/send.py |  4 +++
 bfabric_scripts/docs/changelog.md             |  2 +-
 .../bfabric_scripts/cli/workunit/upload.py    |  6 +++++
 .../operations/workunit/test_upload_files.py  | 11 ++++++++
 tests/bfabric/transfer/generic/test_send.py   | 11 ++++++++
 7 files changed, 59 insertions(+), 3 deletions(-)

diff --git a/bfabric/docs/changelog.md b/bfabric/docs/changelog.md
index e0a0c2af6..1887dcb74 100644
--- a/bfabric/docs/changelog.md
+++ b/bfabric/docs/changelog.md
@@ -13,6 +13,7 @@ Minor breaking changes are still possible in `1.X.Y` but we try to announce them
 
 - `upload_files` accepts `on_hash_progress`, called while each file's MD5 is computed; `bfabric-cli workunit upload` shows it as a progress bar.
 - `upload_files` remembers each file's MD5 by path, size and mtime (`hash_cache`, default `~/.bfabric/hashes.json`), so a retry or resumed upload skips re-hashing unchanged files. `hash_workers` hashes several entries concurrently.
+- `upload_files` accepts `chunk_size` (bytes per tus request) and `send_to_sink` forwards it to the tus mover.
 
 ### Changed
 
diff --git a/bfabric/src/bfabric/operations/workunit/upload.py b/bfabric/src/bfabric/operations/workunit/upload.py
index cd2736fab..720df9d24 100644
--- a/bfabric/src/bfabric/operations/workunit/upload.py
+++ b/bfabric/src/bfabric/operations/workunit/upload.py
@@ -202,6 +202,7 @@ def upload_files(
     resume_cache: Path | None = _USE_DEFAULT_RESUME_CACHE,
     hash_cache: Path | None = _USE_DEFAULT_HASH_CACHE,
     hash_workers: int = 1,
+    chunk_size: int | None = None,
 ) -> UploadSummary:
     """Upload files to a B-Fabric workunit over tus, end to end.
 
@@ -269,6 +270,8 @@ def upload_files(
     :param hash_workers: how many entries are hashed concurrently (``1`` = sequentially). Each entry
         (a file or a directory) is hashed by one worker, so this only helps with several entries.
         ``on_hash_progress`` is then called from worker threads.
+    :param chunk_size: bytes per tus ``PATCH`` request (``None`` uses the mover's default of 4 MiB). Larger
+        chunks mean fewer round trips but more bytes re-sent when a chunk fails.
     :param exclude_names: basenames to skip at any depth (e.g. a sentinel file the caller drops in
         the folder, or ``.DS_Store``). Filter here rather than pre-filtering ``files`` yourself: a
         flat file list loses the directory that gives nested files their relative resource name.
@@ -381,6 +384,7 @@ def upload_files(
                 on_url=on_url,
                 resume_cache=cache,
                 application_id=params.application_id,
+                chunk_size=chunk_size,
             )
     except BaseException:
         # Mark the workunit failed (do NOT delete) so the partial state is diagnosable — see the
@@ -799,6 +803,7 @@ def _transfer_files(
     on_url: FileUrlCallback | None = None,
     resume_cache: ResumeCache | None = None,
     application_id: int | None = None,
+    chunk_size: int | None = None,
 ) -> tuple[list[FileUpload], list[FileFailure]]:
     """Transfer each file over tus, recording per-file success/failure.
 
@@ -839,6 +844,7 @@ def _transfer_files(
                 file_url,
                 resume_cache,
                 _resume_url_for(adopted.get(file_info.name), file_info.name, token_result.tus_endpoint),
+                chunk_size,
             )
         except TransferError as error:
             logger.warning("Upload failed for {}: {}", file_info.name, error)
@@ -863,6 +869,7 @@ def _transfer_one(
     on_url: Callable[[str], None] | None,
     resume_cache: ResumeCache | None,
     resume_url: str | None,
+    chunk_size: int | None = None,
 ) -> None:
     """Send one file, resuming from ``resume_url`` when the caller supplied one.
 
@@ -871,14 +878,30 @@ def _transfer_one(
     rather than a recorded failure. A failure without a resume URL is genuine and propagates.
     """
     try:
-        _ = send_to_sink(sink, file_info.path, creds, on_progress=on_progress, on_url=on_url, resume_url=resume_url)
+        _ = send_to_sink(
+            sink,
+            file_info.path,
+            creds,
+            on_progress=on_progress,
+            on_url=on_url,
+            resume_url=resume_url,
+            chunk_size=chunk_size,
+        )
     except TransferError:
         if resume_url is None:
             raise
         logger.info("Resume URL for {} is no longer usable; restarting the upload.", file_info.name)
         assert resume_cache is not None
         resume_cache.discard(md5=file_info.md5, path=str(file_info.path))
-        _ = send_to_sink(sink, file_info.path, creds, on_progress=on_progress, on_url=on_url, resume_url=None)
+        _ = send_to_sink(
+            sink,
+            file_info.path,
+            creds,
+            on_progress=on_progress,
+            on_url=on_url,
+            resume_url=None,
+            chunk_size=chunk_size,
+        )
 
 
 def _resume_url_for(entry: ResumeEntry | None, filename: str, tus_endpoint: str) -> str | None:
diff --git a/bfabric/src/bfabric/transfer/_generic/send.py b/bfabric/src/bfabric/transfer/_generic/send.py
index b2d1df6a3..9257d88d8 100644
--- a/bfabric/src/bfabric/transfer/_generic/send.py
+++ b/bfabric/src/bfabric/transfer/_generic/send.py
@@ -30,6 +30,7 @@ def send_to_sink(
     on_progress: ProgressCallback | None = None,
     resume_url: str | None = None,
     on_url: UrlCallback | None = None,
+    chunk_size: int | None = None,
 ) -> UploadOutcome:
     """Pushes the bytes of ``src`` to ``sink``.
 
@@ -37,6 +38,8 @@ def send_to_sink(
     resume an interrupted transfer. scp and local copies are not resumable, so ``resume_url`` /
     ``on_url`` are ignored for them and the returned ``UploadOutcome.upload_url`` is ``None``. Raises
     :class:`~bfabric.transfer._generic.errors.TransferError` on failure.
+
+    ``chunk_size`` is the bytes per tus ``PATCH`` request (``None`` uses the mover's default); other sinks ignore it.
     """
     match sink:
         case TransferSinkLocal():
@@ -63,6 +66,7 @@ def send_to_sink(
                 on_progress=on_progress,
                 resume_url=resume_url,
                 on_url=on_url,
+                **({} if chunk_size is None else {"chunk_size": chunk_size}),
             )
 
 
diff --git a/bfabric_scripts/docs/changelog.md b/bfabric_scripts/docs/changelog.md
index 880016ed2..bb9b657f1 100644
--- a/bfabric_scripts/docs/changelog.md
+++ b/bfabric_scripts/docs/changelog.md
@@ -2,7 +2,7 @@
 
 ### Changed
 
-- `bfabric-cli login` switches to the device-code flow over SSH, without a display, or with only a text browser, unless `--no-browser` is given. `bfabric-cli workunit upload` shows a progress bar while hashing, hashes 4 files at once (`--hash-workers`), and reuses checksums of unchanged files from earlier runs (`--no-hash-cache` to disable).
+- `bfabric-cli login` switches to the device-code flow over SSH, without a display, or with only a text browser, unless `--no-browser` is given. `bfabric-cli workunit upload` shows a progress bar while hashing, hashes 4 files at once (`--hash-workers`), and reuses checksums of unchanged files from earlier runs (`--no-hash-cache` to disable). `--chunk-size` (MiB) sets the upload chunk size.
 
 ## \[1.18.0\] - 2026-09-16
 
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py b/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py
index 87ce17222..f074a6620 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py
@@ -58,12 +58,17 @@ class UploadParams(BaseModel):
 
     hash_workers: Annotated[int, cyclopts.Parameter(name="--hash-workers")] = 4
     """How many files to compute checksums for at the same time (``1`` = one at a time)."""
+    chunk_size: Annotated[int | None, cyclopts.Parameter(name="--chunk-size")] = None
+    """Upload chunk size in MiB (``None`` uses the built-in 4). Larger chunks mean fewer round trips, which
+    can raise throughput on fast links, but more data is re-sent when a chunk fails."""
     hash_cache: bool = True
     """Reuse checksums of files whose size and modification time are unchanged since a previous run
     (``~/.bfabric/hashes.json``). Pass ``--no-hash-cache`` to always re-read every file."""
 
     @model_validator(mode="after")
     def _validate_target(self) -> UploadParams:
+        if self.chunk_size is not None and self.chunk_size < 1:
+            raise ValueError("--chunk-size must be at least 1 (MiB).")
         if self.workunit_id is not None:
             if self.workunit_name is not None:
                 raise ValueError("--workunit-name and --workunit-id are mutually exclusive.")
@@ -100,6 +105,7 @@ def cmd_workunit_upload(params: UploadParams, *, client: Bfabric) -> None:
             on_progress=reporter.on_progress if reporter else None,
             on_hash_progress=reporter.on_hash_progress if reporter else None,
             hash_workers=params.hash_workers,
+            chunk_size=params.chunk_size * 1024 * 1024 if params.chunk_size is not None else None,
             hash_cache=compute_hash_cache_path().expanduser() if params.hash_cache else None,
             on_start=reporter.on_start if reporter else None,
             on_file_done=reporter.on_file_done if reporter else None,
diff --git a/tests/bfabric/operations/workunit/test_upload_files.py b/tests/bfabric/operations/workunit/test_upload_files.py
index 6f4db86e2..8654226c1 100644
--- a/tests/bfabric/operations/workunit/test_upload_files.py
+++ b/tests/bfabric/operations/workunit/test_upload_files.py
@@ -241,6 +241,17 @@ def test_cache_is_flushed_when_hashing_is_interrupted(self, tmp_path, mocker, mo
         flush.assert_called_once()
 
 
+class TestChunkSize:
+    @pytest.mark.parametrize("chunk_size", [None, 8 * 1024 * 1024])
+    def test_forwarded_to_the_transfer(self, mock_client, rest, mock_send, chunk_size):
+        rest.create_resources.return_value = _created("a.txt")
+        rest.get_upload_token.return_value = UploadTokenResult(token="tok", tus_endpoint="https://tus/")
+
+        _ = upload_files(mock_client, _params("/src/a.txt"), chunk_size=chunk_size)
+
+        assert mock_send.call_args.kwargs["chunk_size"] == chunk_size
+
+
 class TestExcludeNames:
     def test_forwarded_to_collector(self, mock_client, rest, mock_collect, mock_send):
         rest.create_resources.return_value = _created("src")
diff --git a/tests/bfabric/transfer/generic/test_send.py b/tests/bfabric/transfer/generic/test_send.py
index 6aac3b26e..79cb2d685 100644
--- a/tests/bfabric/transfer/generic/test_send.py
+++ b/tests/bfabric/transfer/generic/test_send.py
@@ -85,6 +85,17 @@ def test_send_tus_delegates_to_mover(mocker, tmp_path):
     assert outcome is sentinel
 
 
+def test_send_tus_forwards_chunk_size(mocker, tmp_path):
+    src = tmp_path / "src.bin"
+    src.write_bytes(b"payload")
+    mock_upload = mocker.patch.object(_tus_mover, "upload_file")
+
+    sink = TransferSinkTus(endpoint="https://tus.example/", metadata={}, token="tus-tok")
+    send_to_sink(sink, src, Credentials(), chunk_size=32 * 1024 * 1024)
+
+    assert mock_upload.call_args.kwargs["chunk_size"] == 32 * 1024 * 1024
+
+
 def test_send_tus_missing_extra_raises_transfererror(mocker, tmp_path):
     # A base install lacks tuspy, so importing the mover fails; the user should get a clear pointer
     # to the extra, not a bare ModuleNotFoundError. Setting the module to None makes the lazy

From f71a7b7d45ebd0d24d5db86b57e6ab95f31dce86 Mon Sep 17 00:00:00 2001
From: Claudio Cannizzaro <claudio.cannizzaro@fgcz.uzh.ch>
Date: Tue, 29 Sep 2026 12:22:14 +0200
Subject: [PATCH 05/12] feat: overall hashing progress with ETA in workunit
 upload

---
 bfabric/docs/changelog.md                     |  2 +-
 .../src/bfabric/operations/workunit/upload.py |  7 +++++
 .../bfabric/transfer/_generic/checksums.py    | 16 +++++++++-
 bfabric_scripts/docs/changelog.md             |  2 +-
 .../bfabric_scripts/cli/workunit/upload.py    | 24 +++++++++++++++
 .../operations/workunit/test_upload_files.py  | 12 ++++++++
 .../transfer/generic/test_checksums.py        | 29 +++++++++++++++++++
 .../cli/workunit/test_cmd_workunit_upload.py  | 21 ++++++++++++++
 8 files changed, 110 insertions(+), 3 deletions(-)

diff --git a/bfabric/docs/changelog.md b/bfabric/docs/changelog.md
index 1887dcb74..0e0029a17 100644
--- a/bfabric/docs/changelog.md
+++ b/bfabric/docs/changelog.md
@@ -12,7 +12,7 @@ Minor breaking changes are still possible in `1.X.Y` but we try to announce them
 ### Added
 
 - `upload_files` accepts `on_hash_progress`, called while each file's MD5 is computed; `bfabric-cli workunit upload` shows it as a progress bar.
-- `upload_files` remembers each file's MD5 by path, size and mtime (`hash_cache`, default `~/.bfabric/hashes.json`), so a retry or resumed upload skips re-hashing unchanged files. `hash_workers` hashes several entries concurrently.
+- `upload_files` remembers each file's MD5 by path, size and mtime (`hash_cache`, default `~/.bfabric/hashes.json`), so a retry or resumed upload skips re-hashing unchanged files. `hash_workers` hashes several entries concurrently. `on_hash_start` reports the total files and bytes before hashing, and a file served from the hash cache is reported as fully hashed.
 - `upload_files` accepts `chunk_size` (bytes per tus request) and `send_to_sink` forwards it to the tus mover.
 
 ### Changed
diff --git a/bfabric/src/bfabric/operations/workunit/upload.py b/bfabric/src/bfabric/operations/workunit/upload.py
index 720df9d24..124e3f90f 100644
--- a/bfabric/src/bfabric/operations/workunit/upload.py
+++ b/bfabric/src/bfabric/operations/workunit/upload.py
@@ -24,6 +24,7 @@
     send_to_sink,
     tus_sink_for_resource,
 )
+from bfabric.transfer._generic.checksums import total_size
 from bfabric.transfer._generic.origin import same_origin
 from bfabric.transfer.hash_cache import HashCache, compute_hash_cache_path
 from bfabric.transfer.resume_cache import ResumeCache, ResumeEntry, compute_resume_cache_path
@@ -194,6 +195,7 @@ def upload_files(
     *,
     on_progress: FileProgressCallback | None = None,
     on_hash_progress: FileProgressCallback | None = None,
+    on_hash_start: UploadStartCallback | None = None,
     on_start: UploadStartCallback | None = None,
     on_file_done: FileDoneCallback | None = None,
     on_url: FileUrlCallback | None = None,
@@ -241,6 +243,9 @@ def upload_files(
     :param on_progress: optional ``(filename, bytes_done, total)`` per-chunk progress callback.
     :param on_hash_progress: optional ``(filename, bytes_done, total)`` callback fired while each file's MD5 is
         computed, before anything is created or transferred.
+    :param on_hash_start: optional ``(total_files, total_bytes)`` callback fired once before any file is hashed,
+        for an overall hashing total. Every file, including one whose MD5 comes from the hash cache, then
+        reaches ``bytes_done == total`` in ``on_hash_progress``.
     :param on_start: optional ``(total_files, total_bytes)`` callback fired once after dedup, just
         before the first transfer (never fired when everything is skipped as a duplicate). It reports
         the post-dedup file set, which is decided before ``create-resources`` runs: should the server
@@ -293,6 +298,8 @@ def upload_files(
     check_upload_scope(client)
     hash_cache_path = compute_hash_cache_path().expanduser() if hash_cache is _USE_DEFAULT_HASH_CACHE else hash_cache
     hashes = HashCache(hash_cache_path) if hash_cache_path is not None else None
+    if on_hash_start is not None:
+        on_hash_start(*total_size([entry.path for entry in params.files], exclude_names=exclude_names))
     try:
         file_infos, policies = _collect_entries(params, exclude_names, on_hash_progress, hashes, hash_workers)
     finally:
diff --git a/bfabric/src/bfabric/transfer/_generic/checksums.py b/bfabric/src/bfabric/transfer/_generic/checksums.py
index bbc83457f..b2b6fa987 100644
--- a/bfabric/src/bfabric/transfer/_generic/checksums.py
+++ b/bfabric/src/bfabric/transfer/_generic/checksums.py
@@ -61,6 +61,16 @@ def resolve_paths(paths: list[Path]) -> list[Path]:
     return result
 
 
+def total_size(paths: list[Path], *, exclude_names: Collection[str] | None = None) -> tuple[int, int]:
+    """The number of files and total bytes :func:`collect_file_infos` would hash for ``paths``.
+
+    Only stats the files, so it is cheap next to hashing them; it lets a caller show an overall progress bar.
+    """
+    excluded = frozenset(exclude_names or ())
+    files = [f for f in resolve_paths(paths) if f.name not in excluded]
+    return len(files), sum(f.stat().st_size for f in files)
+
+
 def collect_file_infos(
     paths: list[Path],
     *,
@@ -114,7 +124,11 @@ def compute_file_info(
     # Stat before reading: a file that changes mid-hash then mismatches on the next lookup.
     stat = path.stat()
     md5 = hash_cache.get(path, size=stat.st_size, mtime_ns=stat.st_mtime_ns) if hash_cache is not None else None
-    if md5 is None:
+    if md5 is not None:
+        # Report the file as fully hashed, so an overall progress total still reaches 100%.
+        if on_hash_progress is not None:
+            on_hash_progress(name, stat.st_size, stat.st_size)
+    else:
         progress = (lambda done, total: on_hash_progress(name, done, total)) if on_hash_progress else None
         md5 = md5_checksum(path, progress)
         if hash_cache is not None:
diff --git a/bfabric_scripts/docs/changelog.md b/bfabric_scripts/docs/changelog.md
index bb9b657f1..beed109e2 100644
--- a/bfabric_scripts/docs/changelog.md
+++ b/bfabric_scripts/docs/changelog.md
@@ -2,7 +2,7 @@
 
 ### Changed
 
-- `bfabric-cli login` switches to the device-code flow over SSH, without a display, or with only a text browser, unless `--no-browser` is given. `bfabric-cli workunit upload` shows a progress bar while hashing, hashes 4 files at once (`--hash-workers`), and reuses checksums of unchanged files from earlier runs (`--no-hash-cache` to disable). `--chunk-size` (MiB) sets the upload chunk size.
+- `bfabric-cli login` switches to the device-code flow over SSH, without a display, or with only a text browser, unless `--no-browser` is given. `bfabric-cli workunit upload` shows a progress bar while hashing, including an overall total and ETA, hashes 4 files at once (`--hash-workers`), and reuses checksums of unchanged files from earlier runs (`--no-hash-cache` to disable). `--chunk-size` (MiB) sets the upload chunk size.
 
 ## \[1.18.0\] - 2026-09-16
 
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py b/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py
index f074a6620..3997e0ece 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import sys
+import threading
 from contextlib import contextmanager
 from pathlib import Path
 from typing import TYPE_CHECKING, Annotated, final
@@ -19,6 +20,7 @@
     TaskProgressColumn,
     TextColumn,
     TimeElapsedColumn,
+    TimeRemainingColumn,
     TransferSpeedColumn,
 )
 
@@ -104,6 +106,7 @@ def cmd_workunit_upload(params: UploadParams, *, client: Bfabric) -> None:
             ),
             on_progress=reporter.on_progress if reporter else None,
             on_hash_progress=reporter.on_hash_progress if reporter else None,
+            on_hash_start=reporter.on_hash_start if reporter else None,
             hash_workers=params.hash_workers,
             chunk_size=params.chunk_size * 1024 * 1024 if params.chunk_size is not None else None,
             hash_cache=compute_hash_cache_path().expanduser() if params.hash_cache else None,
@@ -149,6 +152,12 @@ def __init__(self, *, live: Live, overall: Progress, current: Progress) -> None:
         self._file_tasks: dict[str, TaskID] = {}
         self._total_files = 0
         self._done = 0
+        # Hashing runs on several threads, so its overall total is guarded.
+        self._hash_lock = threading.Lock()
+        self._hash_total_task: TaskID | None = None
+        self._hash_total = 0
+        self._hash_done = 0
+        self._hash_file_done: dict[str, int] = {}
 
     def on_start(self, total_files: int, _total_bytes: int) -> None:
         # _total_bytes is part of the on_start contract but unused: the overall bar is file-count only
@@ -156,6 +165,11 @@ def on_start(self, total_files: int, _total_bytes: int) -> None:
         self._total_files = total_files
         self._overall_task = self._overall.add_task("Overall", total=total_files)
 
+    def on_hash_start(self, _total_files: int, total_bytes: int) -> None:
+        if total_bytes > 0:
+            self._hash_total = total_bytes
+            self._hash_total_task = self._current.add_task("Hashing (all files)", total=total_bytes)
+
     def on_hash_progress(self, filename: str, bytes_done: int, total: int) -> None:
         key = f"hash:{filename}"
         task_id = self._file_tasks.get(key)
@@ -165,6 +179,15 @@ def on_hash_progress(self, filename: str, bytes_done: int, total: int) -> None:
         self._current.update(task_id, completed=bytes_done, total=total)
         if bytes_done >= total:
             self._current.remove_task(self._file_tasks.pop(key))
+        if self._hash_total_task is not None:
+            with self._hash_lock:
+                delta = bytes_done - self._hash_file_done.get(filename, 0)
+                self._hash_file_done[filename] = bytes_done
+                self._hash_done += delta
+                self._current.advance(self._hash_total_task, delta)
+                if self._hash_done >= self._hash_total:
+                    self._current.remove_task(self._hash_total_task)
+                    self._hash_total_task = None
 
     def on_progress(self, filename: str, bytes_done: int, total: int) -> None:
         task_id = self._file_tasks.get(filename)
@@ -209,6 +232,7 @@ def _upload_progress(*, enabled: bool) -> Iterator[_UploadProgressReporter | Non
         TaskProgressColumn(),
         DownloadColumn(),
         TransferSpeedColumn(),
+        TimeRemainingColumn(),
         console=console,
     )
     with Live(Group(overall, current), console=console, transient=True) as live:
diff --git a/tests/bfabric/operations/workunit/test_upload_files.py b/tests/bfabric/operations/workunit/test_upload_files.py
index 8654226c1..ed6132aed 100644
--- a/tests/bfabric/operations/workunit/test_upload_files.py
+++ b/tests/bfabric/operations/workunit/test_upload_files.py
@@ -224,6 +224,18 @@ def test_hash_cache_is_passed_to_the_collector(self, tmp_path, mock_client, rest
 
         assert isinstance(mock_collect.call_args.kwargs["hash_cache"], HashCache)
 
+    def test_on_hash_start_gets_the_totals_before_hashing(self, mocker, mock_client, rest, mock_collect, mock_send):
+        self._setup(rest, "a.txt")
+        mocker.patch("bfabric.operations.workunit.upload.total_size", return_value=(3, 300))
+        events: list[object] = []
+        mock_collect.side_effect = lambda paths, **_kw: events.append("hash") or _file_infos("a.txt")
+
+        _ = upload_files(
+            mock_client, _params("/src/a.txt"), on_hash_start=lambda files, size: events.append((files, size))
+        )
+
+        assert events == [(3, 300), "hash"]
+
     def test_no_hash_cache_passes_none(self, mock_client, rest, mock_collect, mock_send):
         self._setup(rest, "a.txt")
 
diff --git a/tests/bfabric/transfer/generic/test_checksums.py b/tests/bfabric/transfer/generic/test_checksums.py
index 8aba2e50d..c20917789 100644
--- a/tests/bfabric/transfer/generic/test_checksums.py
+++ b/tests/bfabric/transfer/generic/test_checksums.py
@@ -9,6 +9,7 @@
     compute_file_info,
     md5_checksum,
     resolve_paths,
+    total_size,
 )
 
 
@@ -57,6 +58,19 @@ def test_second_call_does_not_re_read_the_file(self, tmp_path: Path, mocker) ->
         spy.assert_not_called()
         assert second == first
 
+    def test_cache_hit_reports_the_file_as_fully_hashed(self, tmp_path: Path) -> None:
+        from bfabric.transfer.hash_cache import HashCache
+
+        f = tmp_path / "data.bin"
+        f.write_bytes(b"some bytes to hash")
+        cache = HashCache(tmp_path / "hashes.json")
+        _ = compute_file_info(f, hash_cache=cache)
+        calls: list[tuple[str, int, int]] = []
+
+        _ = compute_file_info(f, on_hash_progress=lambda *args: calls.append(args), hash_cache=cache)
+
+        assert calls == [("data.bin", 18, 18)]
+
     def test_modified_file_is_re_hashed(self, tmp_path: Path) -> None:
         from bfabric.transfer.hash_cache import HashCache
 
@@ -165,3 +179,18 @@ def test_collect_file_infos_dir_excluded_to_empty_raises(tmp_path: Path) -> None
     # silently creating a workunit with no resources.
     with pytest.raises(ValueError):
         collect_file_infos([d], exclude_names={".marker"})
+
+
+class TestTotalSize:
+    def test_counts_files_and_bytes_across_files_and_directories(self, tmp_path: Path) -> None:
+        (tmp_path / "d").mkdir()
+        (tmp_path / "d" / "a.txt").write_bytes(b"abc")
+        (tmp_path / "b.txt").write_bytes(b"12345")
+
+        assert total_size([tmp_path / "d", tmp_path / "b.txt"]) == (2, 8)
+
+    def test_exclude_names_are_left_out(self, tmp_path: Path) -> None:
+        (tmp_path / "a.txt").write_bytes(b"abc")
+        (tmp_path / ".marker").write_bytes(b"zz")
+
+        assert total_size([tmp_path], exclude_names={".marker"}) == (1, 3)
diff --git a/tests/bfabric_scripts/cli/workunit/test_cmd_workunit_upload.py b/tests/bfabric_scripts/cli/workunit/test_cmd_workunit_upload.py
index c99da0918..ff7b26d01 100644
--- a/tests/bfabric_scripts/cli/workunit/test_cmd_workunit_upload.py
+++ b/tests/bfabric_scripts/cli/workunit/test_cmd_workunit_upload.py
@@ -50,6 +50,26 @@ def test_on_start_creates_overall_task_with_total(self, mocker):
         reporter._overall.add_task.assert_called_once()
         assert reporter._overall.add_task.call_args.kwargs["total"] == 3
 
+    def test_hashing_total_advances_by_deltas_and_disappears_when_complete(self, mocker):
+        reporter = self._reporter(mocker)
+        reporter._current.add_task.side_effect = [1, 2, 3, 4]
+
+        reporter.on_hash_start(2, 150)
+        reporter.on_hash_progress("a.raw", 50, 100)
+        reporter.on_hash_progress("a.raw", 100, 100)
+        reporter.on_hash_progress("b.raw", 50, 50)
+
+        advanced = [call.args[1] for call in reporter._current.advance.call_args_list]
+        assert advanced == [50, 50, 50]
+        reporter._current.remove_task.assert_any_call(1)
+
+    def test_hash_start_without_bytes_adds_no_total(self, mocker):
+        reporter = self._reporter(mocker)
+
+        reporter.on_hash_start(1, 0)
+
+        reporter._current.add_task.assert_not_called()
+
     def test_on_progress_adds_one_task_per_file_and_updates(self, mocker):
         reporter = self._reporter(mocker)
         reporter._current.add_task.side_effect = [10, 20]
@@ -184,6 +204,7 @@ def test_forwards_all_callbacks_when_enabled(self, mocker, summary):
         _, kwargs = upload_files.call_args
         assert kwargs["on_progress"] is reporter.on_progress
         assert kwargs["on_start"] is reporter.on_start
+        assert kwargs["on_hash_start"] is reporter.on_hash_start
         assert kwargs["on_file_done"] is reporter.on_file_done
         assert kwargs["params"].workunit_name == "WU"
 

From aac6a4e1e4af9bcf87430558939673d79a002607 Mon Sep 17 00:00:00 2001
From: Claudio Cannizzaro <claudio.cannizzaro@fgcz.uzh.ch>
Date: Tue, 29 Sep 2026 12:25:02 +0200
Subject: [PATCH 06/12] fix: remember finished files across an interrupted
 upload so resume does not recreate their resources

---
 bfabric/docs/changelog.md                     |  1 +
 .../src/bfabric/operations/workunit/upload.py | 34 ++++++++++++++++---
 bfabric/src/bfabric/transfer/resume_cache.py  | 21 +++++++++++-
 .../operations/workunit/test_upload_files.py  | 31 +++++++++++++++++
 tests/bfabric/transfer/test_resume_cache.py   | 15 ++++++++
 5 files changed, 97 insertions(+), 5 deletions(-)

diff --git a/bfabric/docs/changelog.md b/bfabric/docs/changelog.md
index 0e0029a17..9f3e80fe2 100644
--- a/bfabric/docs/changelog.md
+++ b/bfabric/docs/changelog.md
@@ -22,6 +22,7 @@ Minor breaking changes are still possible in `1.X.Y` but we try to announce them
 ### Fixed
 
 - `upload_files` resuming an interrupted upload no longer fails with `importResourceIds count must match resourceIds count`. Interrupted uploads saved by an earlier version restart from byte 0 instead of resuming.
+- `upload_files` resuming an interrupted run no longer fails with a 409 conflict for files that already finished; they are reported as uploaded and not created again.
 
 ## \[1.23.0\] - 2026-09-16
 
diff --git a/bfabric/src/bfabric/operations/workunit/upload.py b/bfabric/src/bfabric/operations/workunit/upload.py
index 124e3f90f..e2c6de7b5 100644
--- a/bfabric/src/bfabric/operations/workunit/upload.py
+++ b/bfabric/src/bfabric/operations/workunit/upload.py
@@ -364,7 +364,8 @@ def upload_files(
         # A linked resource already points at stored bytes and is created AVAILABLE, so it must be kept
         # out of both the token request and the transfer loop; sending it would push bytes for a
         # resource the server never expects an upload for.
-        transferable = [fi for fi in to_upload if not resources_by_name[fi.name].linked]
+        finished = {name for name, entry in adopted.items() if entry.completed}
+        transferable = [fi for fi in to_upload if not resources_by_name[fi.name].linked and fi.name not in finished]
         links = [
             _as_file_upload(fi.name, resource) for fi in to_upload if (resource := resources_by_name[fi.name]).linked
         ]
@@ -372,13 +373,20 @@ def upload_files(
             logger.info("{} file(s) registered as links to existing content; not transferring them.", len(links))
         uploads: list[FileUpload] = []
         failures: list[FileFailure] = []
+        if finished:
+            logger.info("{} file(s) were already uploaded by the interrupted run.", len(finished))
+            for fi in to_upload:
+                if fi.name in finished:
+                    uploads.append(_as_file_upload(fi.name, resources_by_name[fi.name]))
+                    if on_file_done is not None:
+                        on_file_done(fi.name, True)
         if transferable:
             pending = [resources_by_name[fi.name] for fi in transferable]
             import_resource_ids = [r.import_resource_id for r in pending if r.import_resource_id is not None]
             token_result = rest.get_upload_token(
                 workunit_id, [r.id for r in pending], import_resource_ids, job_id=job_id
             )
-            uploads, failures = _transfer_files(
+            transferred, failures = _transfer_files(
                 transferable,
                 resources_by_name,
                 token_result,
@@ -393,6 +401,7 @@ def upload_files(
                 application_id=params.application_id,
                 chunk_size=chunk_size,
             )
+            uploads += transferred
     except BaseException:
         # Mark the workunit failed (do NOT delete) so the partial state is diagnosable — see the
         # "Failure cleanup pattern" in operations_module.md.
@@ -428,6 +437,8 @@ def upload_files(
                 f"Upload to workunit {workunit_id} succeeded but marking it 'available' failed: {error}",
                 summary,
             ) from error
+    # The run has ended, so its finished files no longer need remembering; a rerun starts afresh.
+    _forget_completed(cache, to_upload, params, container_id)
     return summary
 
 
@@ -478,6 +489,20 @@ def _adopted_uploads(
     return kept
 
 
+def _forget_completed(
+    cache: ResumeCache | None, to_upload: list[FileInfo], params: UploadFilesParams, container_id: int
+) -> None:
+    """Drop the entries of files that finished transferring, once the run that made them has ended."""
+    if cache is None:
+        return
+    for fi in to_upload:
+        entry = cache.lookup(
+            md5=fi.md5, path=str(fi.path), container_id=container_id, application_id=params.application_id
+        )
+        if entry is not None and entry.completed:
+            cache.discard(md5=fi.md5, path=str(fi.path))
+
+
 def _adopted_job_id(adopted: Mapping[str, ResumeEntry]) -> int | None:
     """The tracking job the adopted uploads were started under, if they agree on one."""
     job_ids = {entry.job_id for entry in adopted.values() if entry.job_id is not None}
@@ -860,8 +885,9 @@ def _transfer_files(
                 on_file_done(file_info.name, False)
             continue
         if resume_cache is not None:
-            # The bytes are stored; a kept URL would only resume an upload that is already complete.
-            resume_cache.discard(md5=file_info.md5, path=str(file_info.path))
+            # The bytes are stored, so the URL is spent -- but keep the entry (flagged) until the run
+            # ends, so a later interruption doesn't make the next run create this resource again.
+            resume_cache.mark_completed(md5=file_info.md5, path=str(file_info.path))
         uploads.append(_as_file_upload(file_info.name, resource))
         if on_file_done is not None:
             on_file_done(file_info.name, True)
diff --git a/bfabric/src/bfabric/transfer/resume_cache.py b/bfabric/src/bfabric/transfer/resume_cache.py
index c2d8964a6..9726b4f82 100644
--- a/bfabric/src/bfabric/transfer/resume_cache.py
+++ b/bfabric/src/bfabric/transfer/resume_cache.py
@@ -26,7 +26,7 @@
 import json
 import os
 import time
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from pathlib import Path
 from typing import TYPE_CHECKING, final
 
@@ -91,6 +91,13 @@ class ResumeEntry:
     """
     import_resource_id: int | None = None
     """The import record ``create-resources`` made for the resource; the token request needs it again on resume."""
+    completed: bool = False
+    """The file finished transferring, but its run has not ended yet.
+
+    Kept so a run interrupted later still knows this file is done: without it the next run sees a file
+    with no entry, asks ``create-resources`` for a resource that already exists, and the server
+    answers 409.
+    """
     stored_at: float = 0.0
 
 
@@ -186,6 +193,15 @@ def store(
         )
         self._write(entries)
 
+    def mark_completed(self, *, md5: str, path: str) -> None:
+        """Record that this file's transfer finished, keeping its entry until :meth:`discard`."""
+        entries = self._load()
+        entry = entries.get(_entry_key(md5, path))
+        if entry is None:
+            return
+        entries[_entry_key(md5, path)] = replace(entry, completed=True)
+        self._write(entries)
+
     def discard(self, *, md5: str, path: str) -> None:
         """Forget this file's entry, e.g. once it has transferred successfully."""
         entries = self._load()
@@ -230,6 +246,7 @@ def _load(self) -> dict[str, ResumeEntry]:
             storage_path = fields.get("storage_path")
             job_id = fields.get("job_id")
             import_resource_id = fields.get("import_resource_id")
+            completed = fields.get("completed")
             if (
                 isinstance(url, str)
                 and isinstance(stored_at, int | float)
@@ -247,6 +264,7 @@ def _load(self) -> dict[str, ResumeEntry]:
                     storage_path=storage_path if isinstance(storage_path, str) else None,
                     job_id=job_id if isinstance(job_id, int) else None,
                     import_resource_id=import_resource_id if isinstance(import_resource_id, int) else None,
+                    completed=completed is True,
                     stored_at=float(stored_at),
                 )
         return parsed
@@ -266,6 +284,7 @@ def _write(self, entries: dict[str, ResumeEntry]) -> None:
                     "storage_path": entry.storage_path,
                     "job_id": entry.job_id,
                     "import_resource_id": entry.import_resource_id,
+                    "completed": entry.completed,
                     "stored_at": entry.stored_at,
                 }
                 for md5, entry in entries.items()
diff --git a/tests/bfabric/operations/workunit/test_upload_files.py b/tests/bfabric/operations/workunit/test_upload_files.py
index ed6132aed..57fefd6de 100644
--- a/tests/bfabric/operations/workunit/test_upload_files.py
+++ b/tests/bfabric/operations/workunit/test_upload_files.py
@@ -1153,6 +1153,37 @@ def _report_then_fail(*a, **kw):
         rest.create_resources.assert_called_once()
         assert mock_send.call_args.kwargs.get("resume_url") is None
 
+    def test_finished_files_of_an_interrupted_run_are_not_created_again(self, tmp_path, mock_client, rest, mock_send):
+        """a.txt finished, b.txt was in flight, c.txt never started: only c.txt needs a new resource.
+
+        Creating a.txt's again is a 409 from the server, since its path is already taken.
+        """
+        cache = tmp_path / "resume.json"
+        params = _params("/src/a.txt", "/src/b.txt", "/src/c.txt")
+        created = _created("a.txt", "b.txt", "c.txt")
+        rest.create_resources.return_value = created
+        rest.get_upload_token.return_value = UploadTokenResult(token="tok", tus_endpoint="https://tus/")
+
+        def _first_run(sink, path, *args, **kw):
+            kw["on_url"](f"https://tus/{path.name}")
+            if path.name == "b.txt":
+                raise KeyboardInterrupt
+
+        mock_send.side_effect = _first_run
+        with pytest.raises(KeyboardInterrupt):
+            upload_files(mock_client, params, resume_cache=cache)
+
+        mock_send.reset_mock(side_effect=True)
+        rest.create_resources.reset_mock()
+        rest.get_upload_token.reset_mock()
+        rest.create_resources.return_value = created[2:]
+        summary = upload_files(mock_client, params, resume_cache=cache)
+
+        assert _created_names(rest) == ["c.txt"]
+        assert [call.args[1].name for call in mock_send.call_args_list] == ["b.txt", "c.txt"]
+        assert sorted(u.filename for u in summary.uploads) == ["a.txt", "b.txt", "c.txt"]
+        assert json.loads(cache.read_text())["entries"] == {}
+
     def test_entry_is_discarded_once_the_file_transfers(self, tmp_path, mock_client, rest, mock_send):
         self._setup(rest)
         cache = tmp_path / "resume.json"
diff --git a/tests/bfabric/transfer/test_resume_cache.py b/tests/bfabric/transfer/test_resume_cache.py
index d1de78c87..00c9a8940 100644
--- a/tests/bfabric/transfer/test_resume_cache.py
+++ b/tests/bfabric/transfer/test_resume_cache.py
@@ -114,6 +114,21 @@ def test_import_resource_id_round_trips(self, cache_path, clock):
         entry = _lookup(cache, md5="aaa")
         assert entry is not None and entry.import_resource_id == 812
 
+    def test_mark_completed_keeps_the_entry_and_flags_it(self, cache_path, clock):
+        cache = ResumeCache(cache_path, now=clock)
+        _store(cache, md5="aaa")
+        assert _lookup(cache, md5="aaa").completed is False
+
+        cache.mark_completed(md5="aaa", path=PATH)
+
+        entry = _lookup(ResumeCache(cache_path, now=clock), md5="aaa")
+        assert entry is not None and entry.completed is True and entry.url == URL
+
+    def test_mark_completed_on_an_absent_entry_is_a_no_op(self, cache_path, clock):
+        cache = ResumeCache(cache_path, now=clock)
+        cache.mark_completed(md5="missing", path=PATH)
+        assert not cache_path.exists()
+
 
 class TestInvalidation:
     def test_cross_origin_entry_is_not_returned(self, cache_path, clock):

From 1f8376a33248f4f33d2df559e2566e8d293255e0 Mon Sep 17 00:00:00 2001
From: Claudio Cannizzaro <claudio.cannizzaro@fgcz.uzh.ch>
Date: Tue, 29 Sep 2026 12:34:07 +0200
Subject: [PATCH 07/12] fix: record every created resource for resume so
 unstarted files are not created twice

---
 bfabric/docs/changelog.md                     |  2 +-
 .../src/bfabric/operations/workunit/upload.py | 56 +++++++++++++-----
 bfabric/src/bfabric/transfer/resume_cache.py  | 45 +++++++++++++-
 .../operations/workunit/test_upload_files.py  | 59 ++++++++++++++++---
 tests/bfabric/transfer/test_resume_cache.py   | 25 ++++++++
 5 files changed, 162 insertions(+), 25 deletions(-)

diff --git a/bfabric/docs/changelog.md b/bfabric/docs/changelog.md
index 9f3e80fe2..fd8b3222b 100644
--- a/bfabric/docs/changelog.md
+++ b/bfabric/docs/changelog.md
@@ -22,7 +22,7 @@ Minor breaking changes are still possible in `1.X.Y` but we try to announce them
 ### Fixed
 
 - `upload_files` resuming an interrupted upload no longer fails with `importResourceIds count must match resourceIds count`. Interrupted uploads saved by an earlier version restart from byte 0 instead of resuming.
-- `upload_files` resuming an interrupted run no longer fails with a 409 conflict for files that already finished; they are reported as uploaded and not created again.
+- `upload_files` resuming an interrupted run no longer fails with a 409 conflict: files that already finished are reported as uploaded, and files that never started reuse the resource created for them.
 
 ## \[1.23.0\] - 2026-09-16
 
diff --git a/bfabric/src/bfabric/operations/workunit/upload.py b/bfabric/src/bfabric/operations/workunit/upload.py
index e2c6de7b5..2b92dad1f 100644
--- a/bfabric/src/bfabric/operations/workunit/upload.py
+++ b/bfabric/src/bfabric/operations/workunit/upload.py
@@ -27,7 +27,7 @@
 from bfabric.transfer._generic.checksums import total_size
 from bfabric.transfer._generic.origin import same_origin
 from bfabric.transfer.hash_cache import HashCache, compute_hash_cache_path
-from bfabric.transfer.resume_cache import ResumeCache, ResumeEntry, compute_resume_cache_path
+from bfabric.transfer.resume_cache import PendingUpload, ResumeCache, ResumeEntry, compute_resume_cache_path
 
 if TYPE_CHECKING:
     from collections.abc import Collection
@@ -360,7 +360,27 @@ def upload_files(
         resources_by_name = _adopted_resources(adopted, to_upload)
         if to_create:
             resources = rest.create_resources(workunit_id, to_create)
-            resources_by_name |= _pair_resources_to_files(resources, to_create)
+            created_by_name = _pair_resources_to_files(resources, to_create)
+            resources_by_name |= created_by_name
+            if cache is not None:
+                # Remember them now: they all exist from here on, whether or not this run reaches them.
+                cache.store_pending(
+                    [
+                        PendingUpload(
+                            md5=fi.md5,
+                            path=str(fi.path),
+                            workunit_id=workunit_id,
+                            resource_id=resource.id,
+                            container_id=container_id,
+                            application_id=params.application_id,
+                            storage_path=resource.storage_path,
+                            job_id=job_id,
+                            import_resource_id=resource.import_resource_id,
+                        )
+                        for fi in to_create
+                        if not (resource := created_by_name[fi.name]).linked
+                    ]
+                )
         # A linked resource already points at stored bytes and is created AVAILABLE, so it must be kept
         # out of both the token request and the transfer loop; sending it would push bytes for a
         # resource the server never expects an upload for.
@@ -437,8 +457,8 @@ def upload_files(
                 f"Upload to workunit {workunit_id} succeeded but marking it 'available' failed: {error}",
                 summary,
             ) from error
-    # The run has ended, so its finished files no longer need remembering; a rerun starts afresh.
-    _forget_completed(cache, to_upload, params, container_id)
+    # The run has ended: unless something is left to resume, a rerun starts afresh.
+    _forget_finished_run(cache, to_upload, params, container_id)
     return summary
 
 
@@ -489,18 +509,18 @@ def _adopted_uploads(
     return kept
 
 
-def _forget_completed(
+def _forget_finished_run(
     cache: ResumeCache | None, to_upload: list[FileInfo], params: UploadFilesParams, container_id: int
 ) -> None:
-    """Drop the entries of files that finished transferring, once the run that made them has ended."""
-    if cache is None:
+    """Drop this run's entries once it has ended, unless a saved URL still makes its workunit worth resuming.
+
+    While any URL remains, the next run adopts this workunit, and then needs every entry -- the
+    finished files and the never-started ones too -- to avoid creating their resources a second time.
+    """
+    if cache is None or _has_resumable(cache, to_upload, params, container_id):
         return
     for fi in to_upload:
-        entry = cache.lookup(
-            md5=fi.md5, path=str(fi.path), container_id=container_id, application_id=params.application_id
-        )
-        if entry is not None and entry.completed:
-            cache.discard(md5=fi.md5, path=str(fi.path))
+        cache.discard(md5=fi.md5, path=str(fi.path))
 
 
 def _adopted_job_id(adopted: Mapping[str, ResumeEntry]) -> int | None:
@@ -533,12 +553,18 @@ def _has_resumable(
     params: UploadFilesParams,
     container_id: int,
 ) -> bool:
-    """Whether any file left a resume URL behind, making this workunit worth keeping for a retry."""
+    """Whether any file left an unfinished resume URL behind, making this workunit worth keeping for a retry."""
     if cache is None:
         return False
     return any(
-        cache.lookup(md5=fi.md5, path=str(fi.path), container_id=container_id, application_id=params.application_id)
+        (
+            entry := cache.lookup(
+                md5=fi.md5, path=str(fi.path), container_id=container_id, application_id=params.application_id
+            )
+        )
         is not None
+        and bool(entry.url)
+        and not entry.completed
         for fi in to_upload
     )
 
@@ -943,7 +969,7 @@ def _resume_url_for(entry: ResumeEntry | None, filename: str, tus_endpoint: str)
     The same-origin check lives here rather than in the adoption lookup because the tus endpoint is
     only minted after the workunit to adopt has been chosen; a cross-origin URL costs a fresh upload.
     """
-    if entry is None:
+    if entry is None or not entry.url:
         return None
     if not same_origin(entry.url, tus_endpoint):
         logger.info("Saved URL for {} is cross-origin with {}; starting afresh.", filename, tus_endpoint)
diff --git a/bfabric/src/bfabric/transfer/resume_cache.py b/bfabric/src/bfabric/transfer/resume_cache.py
index 9726b4f82..eecb363e2 100644
--- a/bfabric/src/bfabric/transfer/resume_cache.py
+++ b/bfabric/src/bfabric/transfer/resume_cache.py
@@ -35,7 +35,7 @@
 from bfabric.transfer._generic.origin import same_origin
 
 if TYPE_CHECKING:
-    from collections.abc import Callable
+    from collections.abc import Callable, Sequence
 
 DEFAULT_RESUME_TTL_SECONDS = 2 * 24 * 60 * 60
 """How long a saved URL is considered usable, matching a typical tusd upload-expiry configuration.
@@ -76,6 +76,7 @@ class ResumeEntry:
     """One interrupted upload: where to continue it, and which records it belongs to."""
 
     url: str
+    """Where to continue the upload; empty when its resource exists but no transfer has started."""
     workunit_id: int
     resource_id: int
     container_id: int
@@ -101,6 +102,22 @@ class ResumeEntry:
     stored_at: float = 0.0
 
 
+@final
+@dataclass(frozen=True)
+class PendingUpload:
+    """A resource created for a file whose transfer has not started, so no tus URL exists yet."""
+
+    md5: str
+    path: str
+    workunit_id: int
+    resource_id: int
+    container_id: int
+    application_id: int | None = None
+    storage_path: str | None = None
+    job_id: int | None = None
+    import_resource_id: int | None = None
+
+
 @final
 class ResumeCache:
     """Maps ``file MD5 -> ResumeEntry``, persisted as JSON with 0o600 permissions.
@@ -193,6 +210,32 @@ def store(
         )
         self._write(entries)
 
+    def store_pending(self, uploads: Sequence[PendingUpload]) -> None:
+        """Record resources that exist but have no upload URL yet, as entries with an empty ``url``.
+
+        ``create-resources`` makes them all up front, so a run interrupted before reaching some file
+        leaves resources the server will refuse to create again (409). Without an entry the next run
+        cannot tell those files already have one.
+        """
+        if not uploads:
+            return
+        now = self._now()
+        entries = {key: value for key, value in self._load().items() if not self._expired(value.stored_at)}
+        for upload in uploads:
+            entries[_entry_key(upload.md5, upload.path)] = ResumeEntry(
+                url="",
+                path=upload.path,
+                workunit_id=upload.workunit_id,
+                resource_id=upload.resource_id,
+                container_id=upload.container_id,
+                application_id=upload.application_id,
+                storage_path=upload.storage_path,
+                job_id=upload.job_id,
+                import_resource_id=upload.import_resource_id,
+                stored_at=now,
+            )
+        self._write(entries)
+
     def mark_completed(self, *, md5: str, path: str) -> None:
         """Record that this file's transfer finished, keeping its entry until :meth:`discard`."""
         entries = self._load()
diff --git a/tests/bfabric/operations/workunit/test_upload_files.py b/tests/bfabric/operations/workunit/test_upload_files.py
index 57fefd6de..298138991 100644
--- a/tests/bfabric/operations/workunit/test_upload_files.py
+++ b/tests/bfabric/operations/workunit/test_upload_files.py
@@ -1153,18 +1153,30 @@ def _report_then_fail(*a, **kw):
         rest.create_resources.assert_called_once()
         assert mock_send.call_args.kwargs.get("resume_url") is None
 
-    def test_finished_files_of_an_interrupted_run_are_not_created_again(self, tmp_path, mock_client, rest, mock_send):
-        """a.txt finished, b.txt was in flight, c.txt never started: only c.txt needs a new resource.
+    def test_resuming_never_creates_a_resource_twice(self, tmp_path, mock_client, rest, mock_send):
+        """a.txt finished, b.txt was in flight, c.txt never started -- all three were created up front.
 
-        Creating a.txt's again is a 409 from the server, since its path is already taken.
+        The server answers 409 to a second ``create-resources`` for a path, so a resumed run must
+        not ask again for any of them, including the file that never got a transfer URL.
         """
         cache = tmp_path / "resume.json"
         params = _params("/src/a.txt", "/src/b.txt", "/src/c.txt")
-        created = _created("a.txt", "b.txt", "c.txt")
-        rest.create_resources.return_value = created
+        created = {c.name: c for c in _created("a.txt", "b.txt", "c.txt")}
+        already_created: set[str] = set()
+
+        def _create(_workunit_id, infos):
+            for fi in infos:
+                if fi.name in already_created:
+                    raise BfabricTransferError(f"create-resources REST call failed (409): {fi.name} exists")
+            already_created.update(fi.name for fi in infos)
+            return [created[fi.name] for fi in infos]
+
+        rest.create_resources.side_effect = _create
         rest.get_upload_token.return_value = UploadTokenResult(token="tok", tus_endpoint="https://tus/")
 
         def _first_run(sink, path, *args, **kw):
+            if path.name == "c.txt":
+                raise AssertionError("c.txt must not be reached")
             kw["on_url"](f"https://tus/{path.name}")
             if path.name == "b.txt":
                 raise KeyboardInterrupt
@@ -1174,16 +1186,47 @@ def _first_run(sink, path, *args, **kw):
             upload_files(mock_client, params, resume_cache=cache)
 
         mock_send.reset_mock(side_effect=True)
-        rest.create_resources.reset_mock()
         rest.get_upload_token.reset_mock()
-        rest.create_resources.return_value = created[2:]
         summary = upload_files(mock_client, params, resume_cache=cache)
 
-        assert _created_names(rest) == ["c.txt"]
+        assert rest.create_resources.call_count == 1
         assert [call.args[1].name for call in mock_send.call_args_list] == ["b.txt", "c.txt"]
+        assert mock_send.call_args_list[0].kwargs["resume_url"] == "https://tus/b.txt"
+        assert mock_send.call_args_list[1].kwargs["resume_url"] is None
+        assert rest.get_upload_token.call_args.args[1:3] == ([11, 12], [91, 92])
         assert sorted(u.filename for u in summary.uploads) == ["a.txt", "b.txt", "c.txt"]
         assert json.loads(cache.read_text())["entries"] == {}
 
+    def test_a_failed_run_keeps_the_workunit_adoptable_while_a_resume_url_remains(
+        self, tmp_path, mock_client, rest, mock_send
+    ):
+        """One file failed mid-transfer (URL saved), another failed before it got one: both stay in the
+        workunit, so the retry must adopt both instead of creating either again."""
+        cache = tmp_path / "resume.json"
+        rest.create_resources.return_value = _created("a.txt", "b.txt")
+        rest.get_upload_token.return_value = UploadTokenResult(token="tok", tus_endpoint="https://tus/")
+
+        def _send(sink, path, *args, **kw):
+            if path.name == "a.txt":
+                kw["on_url"]("https://tus/a")
+            raise TransferError("connection reset")
+
+        mock_send.side_effect = _send
+        _ = upload_files(mock_client, _params("/src/a.txt", "/src/b.txt"), resume_cache=cache)
+
+        entries = json.loads(cache.read_text())["entries"]
+        assert len(entries) == 2
+
+    def test_a_run_that_ends_without_any_resume_url_forgets_its_entries(self, tmp_path, mock_client, rest, mock_send):
+        cache = tmp_path / "resume.json"
+        rest.create_resources.return_value = _created("a.txt")
+        rest.get_upload_token.return_value = UploadTokenResult(token="tok", tus_endpoint="https://tus/")
+        mock_send.side_effect = TransferError("refused")
+
+        _ = upload_files(mock_client, _params("/src/a.txt"), resume_cache=cache)
+
+        assert json.loads(cache.read_text())["entries"] == {}
+
     def test_entry_is_discarded_once_the_file_transfers(self, tmp_path, mock_client, rest, mock_send):
         self._setup(rest)
         cache = tmp_path / "resume.json"
diff --git a/tests/bfabric/transfer/test_resume_cache.py b/tests/bfabric/transfer/test_resume_cache.py
index 00c9a8940..1c6bc3541 100644
--- a/tests/bfabric/transfer/test_resume_cache.py
+++ b/tests/bfabric/transfer/test_resume_cache.py
@@ -8,6 +8,7 @@
 
 from bfabric.transfer.resume_cache import (
     DEFAULT_RESUME_TTL_SECONDS,
+    PendingUpload,
     ResumeCache,
     compute_resume_cache_path,
 )
@@ -114,6 +115,30 @@ def test_import_resource_id_round_trips(self, cache_path, clock):
         entry = _lookup(cache, md5="aaa")
         assert entry is not None and entry.import_resource_id == 812
 
+    def test_store_pending_records_resources_without_a_url(self, cache_path, clock):
+        cache = ResumeCache(cache_path, now=clock)
+        cache.store_pending(
+            [
+                PendingUpload(
+                    md5="aaa",
+                    path=PATH,
+                    workunit_id=900,
+                    resource_id=700,
+                    container_id=CONTAINER_ID,
+                    application_id=APPLICATION_ID,
+                    import_resource_id=800,
+                )
+            ]
+        )
+
+        entry = _lookup(ResumeCache(cache_path, now=clock), md5="aaa")
+        assert entry is not None
+        assert (entry.url, entry.workunit_id, entry.resource_id, entry.import_resource_id) == ("", 900, 700, 800)
+
+    def test_store_pending_with_nothing_writes_nothing(self, cache_path, clock):
+        ResumeCache(cache_path, now=clock).store_pending([])
+        assert not cache_path.exists()
+
     def test_mark_completed_keeps_the_entry_and_flags_it(self, cache_path, clock):
         cache = ResumeCache(cache_path, now=clock)
         _store(cache, md5="aaa")

From 178557edb6864ed70899e86f173b4862268b8544 Mon Sep 17 00:00:00 2001
From: Claudio Cannizzaro <claudio.cannizzaro@fgcz.uzh.ch>
Date: Tue, 29 Sep 2026 13:49:00 +0200
Subject: [PATCH 08/12] feat: default workunit upload chunk size to 32 MiB

---
 bfabric/docs/changelog.md                            |  2 +-
 bfabric/src/bfabric/operations/workunit/upload.py    | 12 +++++++++++-
 bfabric_scripts/docs/changelog.md                    |  2 +-
 .../src/bfabric_scripts/cli/workunit/upload.py       |  4 ++--
 .../bfabric/operations/workunit/test_upload_files.py |  6 +++---
 5 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/bfabric/docs/changelog.md b/bfabric/docs/changelog.md
index fd8b3222b..98a87faa7 100644
--- a/bfabric/docs/changelog.md
+++ b/bfabric/docs/changelog.md
@@ -13,7 +13,7 @@ Minor breaking changes are still possible in `1.X.Y` but we try to announce them
 
 - `upload_files` accepts `on_hash_progress`, called while each file's MD5 is computed; `bfabric-cli workunit upload` shows it as a progress bar.
 - `upload_files` remembers each file's MD5 by path, size and mtime (`hash_cache`, default `~/.bfabric/hashes.json`), so a retry or resumed upload skips re-hashing unchanged files. `hash_workers` hashes several entries concurrently. `on_hash_start` reports the total files and bytes before hashing, and a file served from the hash cache is reported as fully hashed.
-- `upload_files` accepts `chunk_size` (bytes per tus request) and `send_to_sink` forwards it to the tus mover.
+- `upload_files` accepts `chunk_size` (bytes per tus request, default 32 MiB instead of the mover's 4 MiB) and `send_to_sink` forwards it to the tus mover.
 
 ### Changed
 
diff --git a/bfabric/src/bfabric/operations/workunit/upload.py b/bfabric/src/bfabric/operations/workunit/upload.py
index 2b92dad1f..05b1d8640 100644
--- a/bfabric/src/bfabric/operations/workunit/upload.py
+++ b/bfabric/src/bfabric/operations/workunit/upload.py
@@ -57,6 +57,14 @@
 caller must still be able to ask for.
 """
 
+DEFAULT_UPLOAD_CHUNK_SIZE: Final = 32 * 1024 * 1024
+"""Bytes per tus ``PATCH`` request when the caller sets none.
+
+Well above the mover's own 4 MiB: on a fast link each round trip through the proxy costs more than the
+bytes it carries, and 32 MiB gets most of the gain. Kept below 64 MiB because a slow link then risks a
+single request outlasting a proxy timeout (Apache defaults to 60 s: 64 MiB needs about 1 MB/s to fit).
+"""
+
 _USE_DEFAULT_HASH_CACHE: Final = Path("<default>")
 """Sentinel for ``hash_cache``, for the same reason as ``_USE_DEFAULT_RESUME_CACHE``."""
 
@@ -275,7 +283,7 @@ def upload_files(
     :param hash_workers: how many entries are hashed concurrently (``1`` = sequentially). Each entry
         (a file or a directory) is hashed by one worker, so this only helps with several entries.
         ``on_hash_progress`` is then called from worker threads.
-    :param chunk_size: bytes per tus ``PATCH`` request (``None`` uses the mover's default of 4 MiB). Larger
+    :param chunk_size: bytes per tus ``PATCH`` request (``None`` uses :data:`DEFAULT_UPLOAD_CHUNK_SIZE`). Larger
         chunks mean fewer round trips but more bytes re-sent when a chunk fails.
     :param exclude_names: basenames to skip at any depth (e.g. a sentinel file the caller drops in
         the folder, or ``.DS_Store``). Filter here rather than pre-filtering ``files`` yourself: a
@@ -294,6 +302,8 @@ def upload_files(
     # 'failed' workunit behind. (The scope is also re-checked at initiate time for direct
     # UploadRestClient callers.)
     require_tus()
+    if chunk_size is None:
+        chunk_size = DEFAULT_UPLOAD_CHUNK_SIZE
     rest = UploadRestClient(client)
     check_upload_scope(client)
     hash_cache_path = compute_hash_cache_path().expanduser() if hash_cache is _USE_DEFAULT_HASH_CACHE else hash_cache
diff --git a/bfabric_scripts/docs/changelog.md b/bfabric_scripts/docs/changelog.md
index beed109e2..7ca769d64 100644
--- a/bfabric_scripts/docs/changelog.md
+++ b/bfabric_scripts/docs/changelog.md
@@ -2,7 +2,7 @@
 
 ### Changed
 
-- `bfabric-cli login` switches to the device-code flow over SSH, without a display, or with only a text browser, unless `--no-browser` is given. `bfabric-cli workunit upload` shows a progress bar while hashing, including an overall total and ETA, hashes 4 files at once (`--hash-workers`), and reuses checksums of unchanged files from earlier runs (`--no-hash-cache` to disable). `--chunk-size` (MiB) sets the upload chunk size.
+- `bfabric-cli login` switches to the device-code flow over SSH, without a display, or with only a text browser, unless `--no-browser` is given. `bfabric-cli workunit upload` shows a progress bar while hashing, including an overall total and ETA, hashes 4 files at once (`--hash-workers`), and reuses checksums of unchanged files from earlier runs (`--no-hash-cache` to disable). `--chunk-size` (MiB) sets the upload chunk size; it defaults to 32 instead of 4, which is faster on fast links.
 
 ## \[1.18.0\] - 2026-09-16
 
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py b/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py
index 3997e0ece..c6fdc617d 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/workunit/upload.py
@@ -61,8 +61,8 @@ class UploadParams(BaseModel):
     hash_workers: Annotated[int, cyclopts.Parameter(name="--hash-workers")] = 4
     """How many files to compute checksums for at the same time (``1`` = one at a time)."""
     chunk_size: Annotated[int | None, cyclopts.Parameter(name="--chunk-size")] = None
-    """Upload chunk size in MiB (``None`` uses the built-in 4). Larger chunks mean fewer round trips, which
-    can raise throughput on fast links, but more data is re-sent when a chunk fails."""
+    """Upload chunk size in MiB (``None`` uses 32). Larger chunks mean fewer round trips, which can raise
+    throughput on fast links, but a slow link may exceed a proxy's request timeout."""
     hash_cache: bool = True
     """Reuse checksums of files whose size and modification time are unchanged since a previous run
     (``~/.bfabric/hashes.json``). Pass ``--no-hash-cache`` to always re-read every file."""
diff --git a/tests/bfabric/operations/workunit/test_upload_files.py b/tests/bfabric/operations/workunit/test_upload_files.py
index 298138991..7f51067d4 100644
--- a/tests/bfabric/operations/workunit/test_upload_files.py
+++ b/tests/bfabric/operations/workunit/test_upload_files.py
@@ -254,14 +254,14 @@ def test_cache_is_flushed_when_hashing_is_interrupted(self, tmp_path, mocker, mo
 
 
 class TestChunkSize:
-    @pytest.mark.parametrize("chunk_size", [None, 8 * 1024 * 1024])
-    def test_forwarded_to_the_transfer(self, mock_client, rest, mock_send, chunk_size):
+    @pytest.mark.parametrize(("chunk_size", "expected"), [(None, 32 * 1024 * 1024), (8 * 1024 * 1024, 8 * 1024 * 1024)])
+    def test_forwarded_to_the_transfer(self, mock_client, rest, mock_send, chunk_size, expected):
         rest.create_resources.return_value = _created("a.txt")
         rest.get_upload_token.return_value = UploadTokenResult(token="tok", tus_endpoint="https://tus/")
 
         _ = upload_files(mock_client, _params("/src/a.txt"), chunk_size=chunk_size)
 
-        assert mock_send.call_args.kwargs["chunk_size"] == chunk_size
+        assert mock_send.call_args.kwargs["chunk_size"] == expected
 
 
 class TestExcludeNames:

From 64a6ecc3fe180708d581158de9d59db79f5626f1 Mon Sep 17 00:00:00 2001
From: Claudio Cannizzaro <claudio.cannizzaro@fgcz.uzh.ch>
Date: Tue, 29 Sep 2026 13:59:38 +0200
Subject: [PATCH 09/12] fix: satisfy basedpyright in new code and undo
 unrelated black reformatting

---
 bfabric/src/bfabric/config/config_data.py        |  1 -
 .../src/bfabric/transfer/_generic/checksums.py   |  8 ++++++--
 bfabric/src/bfabric/transfer/hash_cache.py       | 16 ++++++++--------
 ...fabric_executable_submitter_functionalTest.py |  2 +-
 .../bfabric_executable_submitter_gridengine.py   |  2 +-
 5 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/bfabric/src/bfabric/config/config_data.py b/bfabric/src/bfabric/config/config_data.py
index fb49f631b..6ac011237 100644
--- a/bfabric/src/bfabric/config/config_data.py
+++ b/bfabric/src/bfabric/config/config_data.py
@@ -39,7 +39,6 @@ def _gather_auth_config(cls, values: object) -> object:
         """Accept the flat ``auth_method``/``client_id``/... kwargs the override JSON carries."""
         if not isinstance(values, dict):
             return values
-
         raw = cast("dict[str, object]", values)
         if "auth_config" in raw or not any(raw.get(key) is not None for key in _FLAT_AUTH_KEYS):
             return raw
diff --git a/bfabric/src/bfabric/transfer/_generic/checksums.py b/bfabric/src/bfabric/transfer/_generic/checksums.py
index b2b6fa987..edece8d87 100644
--- a/bfabric/src/bfabric/transfer/_generic/checksums.py
+++ b/bfabric/src/bfabric/transfer/_generic/checksums.py
@@ -129,8 +129,12 @@ def compute_file_info(
         if on_hash_progress is not None:
             on_hash_progress(name, stat.st_size, stat.st_size)
     else:
-        progress = (lambda done, total: on_hash_progress(name, done, total)) if on_hash_progress else None
-        md5 = md5_checksum(path, progress)
+
+        def report(done: int, total: int) -> None:
+            if on_hash_progress is not None:
+                on_hash_progress(name, done, total)
+
+        md5 = md5_checksum(path, report if on_hash_progress is not None else None)
         if hash_cache is not None:
             hash_cache.put(path, size=stat.st_size, mtime_ns=stat.st_mtime_ns, md5=md5)
     return FileInfo(name=name, md5=md5, size=stat.st_size, path=path)
diff --git a/bfabric/src/bfabric/transfer/hash_cache.py b/bfabric/src/bfabric/transfer/hash_cache.py
index 1fdcd9809..e4c0d192a 100644
--- a/bfabric/src/bfabric/transfer/hash_cache.py
+++ b/bfabric/src/bfabric/transfer/hash_cache.py
@@ -16,7 +16,7 @@
 import threading
 import time
 from pathlib import Path
-from typing import TYPE_CHECKING, final
+from typing import TYPE_CHECKING, cast, final
 
 from loguru import logger
 
@@ -102,16 +102,16 @@ def _read(self) -> dict[str, dict[str, object]]:
             raw: object = json.loads(self._path.read_text())  # pyright: ignore[reportAny]
         except (OSError, json.JSONDecodeError):
             return {}
-        if (
-            not isinstance(raw, dict) or raw.get("version") != _FORMAT_VERSION
-        ):  # pyright: ignore[reportUnknownMemberType]
+        if not isinstance(raw, dict):
             return {}
-        entries = raw.get("entries")  # pyright: ignore[reportUnknownMemberType]
+        document = cast("dict[str, object]", raw)
+        if document.get("version") != _FORMAT_VERSION:
+            return {}
+        entries = document.get("entries")
         if not isinstance(entries, dict):
             return {}
-        return {
-            key: entry for key, entry in entries.items() if isinstance(entry, dict)
-        }  # pyright: ignore[reportUnknownVariableType]
+        stored = cast("dict[str, object]", entries)
+        return {key: cast("dict[str, object]", entry) for key, entry in stored.items() if isinstance(entry, dict)}
 
     def _write(self, entries: dict[str, dict[str, object]]) -> bool:
         payload = {"version": _FORMAT_VERSION, "entries": entries}
diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_executable_submitter_functionalTest.py b/bfabric_scripts/src/bfabric_scripts/bfabric_executable_submitter_functionalTest.py
index 0c51f6bea..3dd07fa90 100755
--- a/bfabric_scripts/src/bfabric_scripts/bfabric_executable_submitter_functionalTest.py
+++ b/bfabric_scripts/src/bfabric_scripts/bfabric_executable_submitter_functionalTest.py
@@ -48,7 +48,7 @@ def main() -> None:
         help="external job id is required.",
     )
 
-    options, args = parser.parse_args()
+    (options, args) = parser.parse_args()
 
     if not options.externaljobid:
         parser.error("option '-j' is required.")
diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_executable_submitter_gridengine.py b/bfabric_scripts/src/bfabric_scripts/bfabric_executable_submitter_gridengine.py
index c836d94c0..abd65f066 100755
--- a/bfabric_scripts/src/bfabric_scripts/bfabric_executable_submitter_gridengine.py
+++ b/bfabric_scripts/src/bfabric_scripts/bfabric_executable_submitter_gridengine.py
@@ -49,7 +49,7 @@ def main() -> None:
         help="external job id is required.",
     )
 
-    options, args = parser.parse_args()
+    (options, args) = parser.parse_args()
 
     if not options.externaljobid:
         parser.error("option '-j' is required.")

From 0fe2ede68de8a35f1018d0dba0850c50aff0dde4 Mon Sep 17 00:00:00 2001
From: Claudio Cannizzaro <claudio.cannizzaro@fgcz.uzh.ch>
Date: Tue, 29 Sep 2026 16:55:47 +0200
Subject: [PATCH 10/12] fix: upgrade http base URLs to https for OAuth requests

An environment recorded with an http:// base URL sent OAuth requests in the clear and then
failed on the server's 307 redirect, which httpx does not follow. Loopback hosts are left alone.
---
 bfabric/docs/changelog.md                     |  4 +++
 bfabric/src/bfabric/oauth/__init__.py         |  3 +-
 bfabric/src/bfabric/oauth/_device_code.py     |  4 +--
 bfabric/src/bfabric/oauth/_endpoints.py       | 22 ++++++++++--
 bfabric/src/bfabric/oauth/_registration.py    |  3 +-
 bfabric/src/bfabric/oauth/_token_exchange.py  |  4 +--
 bfabric/src/bfabric/oauth/_url_token.py       |  3 +-
 bfabric_scripts/docs/changelog.md             |  4 +++
 .../src/bfabric_scripts/cli/login/_common.py  | 17 +++++++++-
 tests/bfabric/oauth/test_device_code.py       |  8 +++++
 tests/bfabric/oauth/test_endpoints.py         | 34 ++++++++++++++++++-
 .../cli/login/test_login_common.py            | 11 ++++++
 12 files changed, 105 insertions(+), 12 deletions(-)

diff --git a/bfabric/docs/changelog.md b/bfabric/docs/changelog.md
index 98a87faa7..924187497 100644
--- a/bfabric/docs/changelog.md
+++ b/bfabric/docs/changelog.md
@@ -9,6 +9,10 @@ Minor breaking changes are still possible in `1.X.Y` but we try to announce them
 
 ## \[Unreleased\]
 
+### Fixed
+
+- OAuth requests to an instance recorded with an `http://` base URL are sent over `https://` instead of failing on the server's redirect (loopback hosts excepted). `bfabric.oauth.https_base_url` exposes the upgrade.
+
 ### Added
 
 - `upload_files` accepts `on_hash_progress`, called while each file's MD5 is computed; `bfabric-cli workunit upload` shows it as a progress bar.
diff --git a/bfabric/src/bfabric/oauth/__init__.py b/bfabric/src/bfabric/oauth/__init__.py
index c79b1ae3d..751aece38 100644
--- a/bfabric/src/bfabric/oauth/__init__.py
+++ b/bfabric/src/bfabric/oauth/__init__.py
@@ -3,7 +3,7 @@
 # Importing this module pulls authlib, joserfc and httpx; ``import bfabric`` alone does not.
 from bfabric.oauth._credential_provider import OAuthCredentialProvider
 from bfabric.oauth._device_code import device_code_login
-from bfabric.oauth._endpoints import token_url
+from bfabric.oauth._endpoints import https_base_url, token_url
 from bfabric.oauth._pkce import AuthorizationRequest, exchange_code, graphical_browser_available, pkce_login
 from bfabric.oauth._registration import (
     delete_client,
@@ -27,6 +27,7 @@
     "device_code_login",
     "exchange_code",
     "graphical_browser_available",
+    "https_base_url",
     "pkce_login",
     "read_client",
     "register_client",
diff --git a/bfabric/src/bfabric/oauth/_device_code.py b/bfabric/src/bfabric/oauth/_device_code.py
index 398393fd7..59d5ce89a 100644
--- a/bfabric/src/bfabric/oauth/_device_code.py
+++ b/bfabric/src/bfabric/oauth/_device_code.py
@@ -10,7 +10,7 @@
 from loguru import logger
 
 from bfabric.errors import BfabricOAuthError, raise_if_unavailable
-from bfabric.oauth._endpoints import token_url
+from bfabric.oauth._endpoints import https_base_url, token_url
 
 if TYPE_CHECKING:
     from bfabric.config.base_url import BaseUrl
@@ -31,7 +31,7 @@ def _request_device_code(
     logger.debug("Requesting device code from {}", base_url)
     with raise_if_unavailable(base_url):
         response = httpx.post(
-            f"{base_url}/rest/oauth/device_authorization",
+            f"{https_base_url(base_url)}/rest/oauth/device_authorization",
             data={
                 "client_id": client_id,
                 "scope": scope,
diff --git a/bfabric/src/bfabric/oauth/_endpoints.py b/bfabric/src/bfabric/oauth/_endpoints.py
index 55a3e2b97..351d7ad5e 100644
--- a/bfabric/src/bfabric/oauth/_endpoints.py
+++ b/bfabric/src/bfabric/oauth/_endpoints.py
@@ -6,11 +6,27 @@
 
 from __future__ import annotations
 
-from urllib.parse import urlencode
+from urllib.parse import urlencode, urlsplit, urlunsplit
 
 from bfabric.config.base_url import BaseUrl
 
 
+_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"})
+
+
+def https_base_url(base_url: str) -> str:
+    """*base_url* with ``http`` upgraded to ``https``, except for a loopback host (local development).
+
+    OAuth requests carry codes, secrets and refresh tokens, so they must not go out in the clear. A
+    config that still records ``http://`` would otherwise send the first request unencrypted and then
+    fail, because the server answers with a redirect that ``httpx`` does not follow.
+    """
+    parts = urlsplit(base_url)
+    if parts.scheme != "http" or parts.hostname in _LOOPBACK_HOSTS:
+        return base_url
+    return urlunsplit(parts._replace(scheme="https"))
+
+
 def authorize_url(
     base_url: str,
     *,
@@ -36,9 +52,9 @@ def authorize_url(
             "scope": scope,
         }
     )
-    return f"{BaseUrl(base_url)}/rest/oauth/authorize?{query}"
+    return f"{https_base_url(BaseUrl(base_url))}/rest/oauth/authorize?{query}"
 
 
 def token_url(base_url: str) -> str:
     """The token endpoint that issues, exchanges and refreshes tokens."""
-    return f"{BaseUrl(base_url)}/rest/oauth/token"
+    return f"{https_base_url(BaseUrl(base_url))}/rest/oauth/token"
diff --git a/bfabric/src/bfabric/oauth/_registration.py b/bfabric/src/bfabric/oauth/_registration.py
index 2820100cb..dcd887936 100644
--- a/bfabric/src/bfabric/oauth/_registration.py
+++ b/bfabric/src/bfabric/oauth/_registration.py
@@ -8,6 +8,7 @@
 from loguru import logger
 
 from bfabric.errors import raise_if_unavailable
+from bfabric.oauth._endpoints import https_base_url
 
 if TYPE_CHECKING:
     from bfabric.bfabric import Bfabric
@@ -60,7 +61,7 @@ def register_client(
     :param grant_types: Explicit list of grant types to request (overrides the default)
     :returns: Registration response containing ``client_id``, ``client_secret``, etc.
     """
-    url = f"{base_url}/rest/oauth/register"
+    url = f"{https_base_url(base_url)}/rest/oauth/register"
     resolved_grant_types = grant_types if grant_types is not None else _default_grant_types(service_user)
     body: dict[str, object] = {
         "client_name": client_name,
diff --git a/bfabric/src/bfabric/oauth/_token_exchange.py b/bfabric/src/bfabric/oauth/_token_exchange.py
index 710899c50..e46885809 100644
--- a/bfabric/src/bfabric/oauth/_token_exchange.py
+++ b/bfabric/src/bfabric/oauth/_token_exchange.py
@@ -15,7 +15,7 @@
 
 from bfabric.errors import raise_if_unavailable
 
-from bfabric.oauth._endpoints import token_url
+from bfabric.oauth._endpoints import https_base_url, token_url
 from bfabric.oauth._url_token import UrlTokenContext
 
 if TYPE_CHECKING:
@@ -66,7 +66,7 @@ def introspect_token(
 
     :raises httpx.HTTPStatusError: On non-2xx responses
     """
-    url = f"{base_url}/rest/oauth/introspect"
+    url = f"{https_base_url(base_url)}/rest/oauth/introspect"
     logger.debug("Introspecting token at {}", url)
     with raise_if_unavailable(base_url):
         response = httpx.post(
diff --git a/bfabric/src/bfabric/oauth/_url_token.py b/bfabric/src/bfabric/oauth/_url_token.py
index a4d49d809..56c953f5b 100644
--- a/bfabric/src/bfabric/oauth/_url_token.py
+++ b/bfabric/src/bfabric/oauth/_url_token.py
@@ -13,6 +13,7 @@
 
 from bfabric.config.base_url import BaseUrl
 from bfabric.errors import raise_if_unavailable
+from bfabric.oauth._endpoints import https_base_url
 from pydantic import BaseModel, ConfigDict, Field
 
 
@@ -73,7 +74,7 @@ def _fetch_jwks(base_url: BaseUrl) -> dict[str, object]:
                 return jwks
 
     logger.debug("Fetching JWKS from {}", base_url)
-    url = f"{base_url}/rest/oauth/jwks"
+    url = f"{https_base_url(base_url)}/rest/oauth/jwks"
     with raise_if_unavailable(base_url):
         response = httpx.get(url, timeout=30)
     _ = response.raise_for_status()
diff --git a/bfabric_scripts/docs/changelog.md b/bfabric_scripts/docs/changelog.md
index 7ca769d64..65d49c092 100644
--- a/bfabric_scripts/docs/changelog.md
+++ b/bfabric_scripts/docs/changelog.md
@@ -1,5 +1,9 @@
 ## \[Unreleased\]
 
+### Fixed
+
+- `bfabric-cli login` and `auth device-code` no longer fail on an environment recorded with an `http://` base URL; they use `https://` and save the corrected URL.
+
 ### Changed
 
 - `bfabric-cli login` switches to the device-code flow over SSH, without a display, or with only a text browser, unless `--no-browser` is given. `bfabric-cli workunit upload` shows a progress bar while hashing, including an overall total and ETA, hashes 4 files at once (`--hash-workers`), and reuses checksums of unchanged files from earlier runs (`--no-hash-cache` to disable). `--chunk-size` (MiB) sets the upload chunk size; it defaults to 32 instead of 4, which is faster on fast links.
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/login/_common.py b/bfabric_scripts/src/bfabric_scripts/cli/login/_common.py
index 8c84692ae..00d8ee799 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/login/_common.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/login/_common.py
@@ -18,6 +18,7 @@
 from bfabric_scripts.cli.interactive import confirm, is_interactive, select_choice, select_or_input, text_input
 from bfabric_scripts.cli.login._constants import SCOPE_PRESETS, SCOPE_PRESETS_BY_NAME
 from bfabric.config import BaseUrl
+from bfabric.oauth import https_base_url
 from bfabric_scripts.cli.login._urls import KNOWN_INSTANCES, normalize_base_url
 
 if TYPE_CHECKING:
@@ -69,7 +70,21 @@ def _pick_or_type(message: str, labels: dict[str, str], prompt: str) -> str | No
 
 
 def resolve_base_url(base_url: str | None, env: EnvironmentConfig | None) -> BaseUrl | None:
-    """Resolve the instance URL: explicit, else the environment's recorded one, else a picker."""
+    """Resolve the instance URL: explicit, else the environment's recorded one, else a picker.
+
+    A plain ``http`` URL is upgraded to ``https`` (loopback excepted), so a login never sends its OAuth
+    requests unencrypted and the environment is saved with the corrected URL.
+    """
+    resolved = _resolve_raw_base_url(base_url, env)
+    if resolved is None:
+        return None
+    secure = BaseUrl(https_base_url(resolved))
+    if secure != resolved:
+        print(f"Using {secure} instead of {resolved}: OAuth requires https.", file=sys.stderr)
+    return secure
+
+
+def _resolve_raw_base_url(base_url: str | None, env: EnvironmentConfig | None) -> BaseUrl | None:
     if base_url is not None:
         return normalize_base_url(base_url)
     if env is not None:
diff --git a/tests/bfabric/oauth/test_device_code.py b/tests/bfabric/oauth/test_device_code.py
index cd21b2072..a520566bb 100644
--- a/tests/bfabric/oauth/test_device_code.py
+++ b/tests/bfabric/oauth/test_device_code.py
@@ -41,6 +41,14 @@ def test_posts_correct_params(self, mocker):
         )
         assert result == mock_response.json.return_value
 
+    def test_upgrades_a_plain_http_base_url(self, mocker):
+        """A recorded ``http://`` URL would otherwise be sent in the clear, then fail on the server's redirect."""
+        mock_post = mocker.patch("bfabric.oauth._device_code.httpx.post")
+
+        _request_device_code("http://example.com/bfabric", client_id="c", scope="api:read")
+
+        assert mock_post.call_args.args[0] == "https://example.com/bfabric/rest/oauth/device_authorization"
+
     def test_raises_on_http_error(self, mocker):
         mock_response = mocker.MagicMock()
         mock_response.raise_for_status.side_effect = httpx.HTTPStatusError(
diff --git a/tests/bfabric/oauth/test_endpoints.py b/tests/bfabric/oauth/test_endpoints.py
index 2508d8f9f..ee54f9b05 100644
--- a/tests/bfabric/oauth/test_endpoints.py
+++ b/tests/bfabric/oauth/test_endpoints.py
@@ -3,7 +3,8 @@
 import pytest
 
 from bfabric.config.base_url import BaseUrl
-from bfabric.oauth import token_url
+from bfabric.oauth import https_base_url, token_url
+from bfabric.oauth._endpoints import authorize_url
 
 
 class TestTokenUrl:
@@ -17,3 +18,34 @@ def test_validates_the_base_url(self):
         # Only that the validation happens -- what BaseUrl accepts is tested where BaseUrl lives.
         with pytest.raises(ValueError):
             token_url("https://example.com/notbfabric")
+
+
+class TestHttpsBaseUrl:
+    def test_upgrades_http(self):
+        assert https_base_url("http://fgcz-bfabric.uzh.ch/bfabric") == "https://fgcz-bfabric.uzh.ch/bfabric"
+
+    def test_keeps_the_port(self):
+        assert https_base_url("http://example.com:8080/bfabric") == "https://example.com:8080/bfabric"
+
+    def test_leaves_https_alone(self):
+        assert https_base_url("https://example.com/bfabric") == "https://example.com/bfabric"
+
+    @pytest.mark.parametrize(
+        "url", ["http://localhost:8080/bfabric", "http://127.0.0.1:8000/bfabric", "http://[::1]:8000/bfabric"]
+    )
+    def test_leaves_loopback_alone_for_local_development(self, url):
+        assert https_base_url(url) == url
+
+    def test_token_url_uses_https(self):
+        assert token_url("http://example.com/bfabric") == "https://example.com/bfabric/rest/oauth/token"
+
+    def test_authorize_url_uses_https(self):
+        url = authorize_url(
+            "http://example.com/bfabric",
+            client_id="c",
+            redirect_uri="http://127.0.0.1:1/callback",
+            code_challenge="x",
+            state="s",
+            scope="api:read",
+        )
+        assert url.startswith("https://example.com/bfabric/rest/oauth/authorize?")
diff --git a/tests/bfabric_scripts/cli/login/test_login_common.py b/tests/bfabric_scripts/cli/login/test_login_common.py
index e6f4ba3f7..3d53a4f87 100644
--- a/tests/bfabric_scripts/cli/login/test_login_common.py
+++ b/tests/bfabric_scripts/cli/login/test_login_common.py
@@ -186,6 +186,17 @@ def test_falls_back_to_the_recorded_base_url(self):
         env = EnvironmentConfig.model_validate({"base_url": "https://recorded.example.com/bfabric"})
         assert resolve_base_url(None, env) == "https://recorded.example.com/bfabric"
 
+    def test_recorded_http_url_is_upgraded_to_https(self, capsys):
+        env = EnvironmentConfig.model_validate({"base_url": "http://recorded.example.com/bfabric"})
+        assert resolve_base_url(None, env) == "https://recorded.example.com/bfabric"
+        assert "https" in capsys.readouterr().err
+
+    def test_explicit_http_url_is_upgraded_to_https(self):
+        assert resolve_base_url("http://typed.example.com/bfabric", None) == "https://typed.example.com/bfabric"
+
+    def test_explicit_loopback_http_url_is_kept(self):
+        assert resolve_base_url("http://localhost:8000/bfabric", None) == "http://localhost:8000/bfabric"
+
     def test_explicit_value_wins_over_the_recorded_one(self):
         env = EnvironmentConfig.model_validate({"base_url": "https://recorded.example.com/bfabric"})
         assert resolve_base_url("https://typed.example.com/bfabric", env) == "https://typed.example.com/bfabric"

From ebd7c8c45d188f470253f12e9b21f6e9bb29be2a Mon Sep 17 00:00:00 2001
From: Claudio Cannizzaro <claudio.cannizzaro@fgcz.uzh.ch>
Date: Mon, 5 Oct 2026 10:35:19 +0200
Subject: [PATCH 11/12] docs: fix basedpyright nox session names in AGENTS.md

The sessions are parametrised as basedpyright(package='<pkg>'); the bare
basedpyright(<pkg>) form is reported by nox as "Sessions not found".
---
 AGENTS.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md
index 52eee61bb..2b31fa7c4 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -50,9 +50,9 @@ Run each package's suite in a **separate** `pytest` invocation, as nox does —
 
 ### Type Checking
 ```bash
-nox -s basedpyright(bfabric)
-nox -s basedpyright(bfabric_scripts)
-nox -s basedpyright(bfabric_app_runner)
+nox -s "basedpyright(package='bfabric')"
+nox -s "basedpyright(package='bfabric_scripts')"
+nox -s "basedpyright(package='bfabric_app_runner')"
 ```
 
 ### Linting and formatting

From 77defcfc45cf454429b06899e52d8830f4524a62 Mon Sep 17 00:00:00 2001
From: Claudio Cannizzaro <claudio.cannizzaro@fgcz.uzh.ch>
Date: Mon, 5 Oct 2026 10:44:02 +0200
Subject: [PATCH 12/12] chore(release): bfabric 1.24.0, bfabric_scripts 1.19.0,
 bfabric_app_runner 0.9.1

Graduate each [Unreleased] section to its dated version heading and bump the
bfabric floor in the two dependent packages, following each one's established
pattern (scripts <2, app_runner pinned to the minor). Merge the two Fixed
subsections in the bfabric entry into one.

Release mechanics only; no code changes.
---
 bfabric/docs/changelog.md            | 5 ++---
 bfabric/pyproject.toml               | 2 +-
 bfabric_app_runner/docs/changelog.md | 6 ++++++
 bfabric_app_runner/pyproject.toml    | 4 ++--
 bfabric_scripts/docs/changelog.md    | 2 ++
 bfabric_scripts/pyproject.toml       | 4 ++--
 6 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/bfabric/docs/changelog.md b/bfabric/docs/changelog.md
index 924187497..baf966e6a 100644
--- a/bfabric/docs/changelog.md
+++ b/bfabric/docs/changelog.md
@@ -9,9 +9,7 @@ Minor breaking changes are still possible in `1.X.Y` but we try to announce them
 
 ## \[Unreleased\]
 
-### Fixed
-
-- OAuth requests to an instance recorded with an `http://` base URL are sent over `https://` instead of failing on the server's redirect (loopback hosts excepted). `bfabric.oauth.https_base_url` exposes the upgrade.
+## \[1.24.0\] - 2026-10-05
 
 ### Added
 
@@ -27,6 +25,7 @@ Minor breaking changes are still possible in `1.X.Y` but we try to announce them
 
 - `upload_files` resuming an interrupted upload no longer fails with `importResourceIds count must match resourceIds count`. Interrupted uploads saved by an earlier version restart from byte 0 instead of resuming.
 - `upload_files` resuming an interrupted run no longer fails with a 409 conflict: files that already finished are reported as uploaded, and files that never started reuse the resource created for them.
+- OAuth requests to an instance recorded with an `http://` base URL are sent over `https://` instead of failing on the server's redirect (loopback hosts excepted). `bfabric.oauth.https_base_url` exposes the upgrade.
 
 ## \[1.23.0\] - 2026-09-16
 
diff --git a/bfabric/pyproject.toml b/bfabric/pyproject.toml
index af4461685..ceeb0b6f4 100644
--- a/bfabric/pyproject.toml
+++ b/bfabric/pyproject.toml
@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 name = "bfabric"
 description = "Python client for the B-Fabric API"
 readme = "README.md"
-version = "1.23.0"
+version = "1.24.0"
 license = { text = "GPL-3.0" }
 authors = [
     { name = "Christian Panse", email = "cp@fgcz.ethz.ch" },
diff --git a/bfabric_app_runner/docs/changelog.md b/bfabric_app_runner/docs/changelog.md
index b38b2aa1a..0a48465dc 100644
--- a/bfabric_app_runner/docs/changelog.md
+++ b/bfabric_app_runner/docs/changelog.md
@@ -4,6 +4,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## \[Unreleased\]
 
+## \[0.9.1\] - 2026-10-05
+
+### Changed
+
+- Require `bfabric>=1.24.0,<1.25`.
+
 ## \[0.9.0\] - 2026-09-16
 
 ### Changed
diff --git a/bfabric_app_runner/pyproject.toml b/bfabric_app_runner/pyproject.toml
index 2a9a1976b..c777e465b 100644
--- a/bfabric_app_runner/pyproject.toml
+++ b/bfabric_app_runner/pyproject.toml
@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 name = "bfabric_app_runner"
 description = "Application runner for B-Fabric apps"
 readme = "README.md"
-version = "0.9.0"
+version = "0.9.1"
 license = { text = "GPL-3.0" }
 authors = [
     { name = "Leonardo Schwarz", email = "leonardo.schwarz@fgcz.ethz.ch" },
@@ -14,7 +14,7 @@ authors = [
 ]
 requires-python = ">=3.12"
 dependencies = [
-    "bfabric>=1.23.0,<1.24",
+    "bfabric>=1.24.0,<1.25",
     "cyclopts>=4.0,<5.0",
     "pydantic>=2.12.3,<3.0",
     "glom>=24.11,<25.0",
diff --git a/bfabric_scripts/docs/changelog.md b/bfabric_scripts/docs/changelog.md
index 65d49c092..ed8a47102 100644
--- a/bfabric_scripts/docs/changelog.md
+++ b/bfabric_scripts/docs/changelog.md
@@ -1,5 +1,7 @@
 ## \[Unreleased\]
 
+## \[1.19.0\] - 2026-10-05
+
 ### Fixed
 
 - `bfabric-cli login` and `auth device-code` no longer fail on an environment recorded with an `http://` base URL; they use `https://` and save the corrected URL.
diff --git a/bfabric_scripts/pyproject.toml b/bfabric_scripts/pyproject.toml
index 28b7ce756..8089d6924 100644
--- a/bfabric_scripts/pyproject.toml
+++ b/bfabric_scripts/pyproject.toml
@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 name = "bfabric_scripts"
 description = "Python command line scripts for the B-Fabric API"
 readme = "README.md"
-version = "1.18.0"
+version = "1.19.0"
 authors = [
     { name = "Christian Panse", email = "cp@fgcz.ethz.ch" },
     { name = "Claudio Cannizzaro", email = "claudio.cannizzaro@fgcz.uzh.ch" },
@@ -18,7 +18,7 @@ authors = [
 ]
 
 dependencies = [
-    "bfabric[transfer]>=1.23.0,<2",
+    "bfabric[transfer]>=1.24.0,<2",
     "cyclopts>=4.0,<5.0",
     "Flask>=3.0.3,<4.0",
     "questionary>=2.0,<3",