From fe107c4c4807c94c4489364a111e424080f67ee8 Mon Sep 17 00:00:00 2001 From: Gerald Berger Date: Wed, 14 Oct 2026 14:49:28 +0200 Subject: [PATCH 1/3] Fix UB: ArrowBatchInner freed via wrong pointer type in a test oversized_batch_does_not_deadlock_byte_sem freed batch.batch.rust_ptr by casting it to *mut Vec, but record_batch_to_c_ffi allocates it as Box (two Arrow FFI structs) -- see iceberg_arrow_batch_free, which does the correct cast a few hundred lines away in table.rs. Dropping it as the wrong type is undefined behavior: it deallocates with a layout that doesn't match the real allocation. This was latent -- rustc 1.98.1 (CI's last known-good run) didn't catch it, but rustc 1.99.0 (CI's floating `stable` toolchain today) added a stricter allocator precondition check that now aborts on it with "NonNull::new_unchecked requires that the pointer is non-null". Not triggered by any functional change; any PR running CI today would hit it. Co-Authored-By: Claude Sonnet 5 --- iceberg_rust_ffi/src/nested_pipeline.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/iceberg_rust_ffi/src/nested_pipeline.rs b/iceberg_rust_ffi/src/nested_pipeline.rs index 6f58205..172701d 100644 --- a/iceberg_rust_ffi/src/nested_pipeline.rs +++ b/iceberg_rust_ffi/src/nested_pipeline.rs @@ -847,8 +847,18 @@ mod tests { batch.byte_sem.add_permits(batch.clamped_byte_len); assert_eq!(byte_sem.available_permits(), byte_budget); - // Free the IPC payload allocated by serialize_record_batch. - unsafe { drop(Box::from_raw(batch.batch.rust_ptr as *mut Vec)) }; + // Free the ArrowBatchInner allocated by record_batch_to_c_ffi (see + // `iceberg_arrow_batch_free`, which does the same thing at the FFI + // boundary). The prior `*mut Vec` cast here was wrong: `rust_ptr` + // is a `Box`, not a serialized byte buffer, and + // dropping it as the wrong type is undefined behavior (freeing with + // a layout that doesn't match the real allocation) -- rustc 1.99's + // new allocator precondition checks started catching it as a SIGABRT. + unsafe { + drop(Box::from_raw( + batch.batch.rust_ptr as *mut crate::table::ArrowBatchInner, + )) + }; // Drain finishes cleanly once the stream is exhausted. drop(rx); From 69058b048f77375ab4c79e5ff3e10d94adcf5cf0 Mon Sep 17 00:00:00 2001 From: Gerald Berger Date: Wed, 14 Oct 2026 14:16:17 +0200 Subject: [PATCH 2/3] Fix CI: bump crate version, replace deprecated fetch_update check-version-bump requires a version bump whenever iceberg_rust_ffi's own source changes. Also replaces fetch_update with try_update in pipeline_stats.rs (unrelated to this PR's own changes, but the CI toolchain now deprecates it under -D warnings, breaking clippy/build for every PR against this repo). Co-Authored-By: Claude Sonnet 5 --- iceberg_rust_ffi/Cargo.lock | 2 +- iceberg_rust_ffi/Cargo.toml | 2 +- iceberg_rust_ffi/src/pipeline_stats.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/iceberg_rust_ffi/Cargo.lock b/iceberg_rust_ffi/Cargo.lock index 7d0c377..803bbe3 100644 --- a/iceberg_rust_ffi/Cargo.lock +++ b/iceberg_rust_ffi/Cargo.lock @@ -1910,7 +1910,7 @@ dependencies = [ [[package]] name = "iceberg_rust_ffi" -version = "0.9.5" +version = "0.9.6" dependencies = [ "anyhow", "arrow-array", diff --git a/iceberg_rust_ffi/Cargo.toml b/iceberg_rust_ffi/Cargo.toml index 93b18b7..5b5efe5 100644 --- a/iceberg_rust_ffi/Cargo.toml +++ b/iceberg_rust_ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iceberg_rust_ffi" -version = "0.9.5" +version = "0.9.6" edition = "2021" [lib] diff --git a/iceberg_rust_ffi/src/pipeline_stats.rs b/iceberg_rust_ffi/src/pipeline_stats.rs index ef82542..925746e 100644 --- a/iceberg_rust_ffi/src/pipeline_stats.rs +++ b/iceberg_rust_ffi/src/pipeline_stats.rs @@ -164,7 +164,7 @@ impl PipelineStats { /// caused by lingering tasks from a previous scan releasing after a reset(). pub(crate) fn track_buffer_release(&self, bytes: u64) { self.buffered_bytes - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| { + .try_update(Ordering::Relaxed, Ordering::Relaxed, |v| { Some(v.saturating_sub(bytes)) }) .ok(); From 8036ff54a2d3a78d6cb8fc754bc36dc219df6d2e Mon Sep 17 00:00:00 2001 From: Gerald Berger Date: Wed, 14 Oct 2026 14:32:41 +0200 Subject: [PATCH 3/3] Fix CI: build minio/mc locally instead of pulling unpullable images quay.io/minio/minio and quay.io/minio/mc (like their Docker Hub equivalents) are no longer pullable -- MinIO discontinued prebuilt community images -- which was failing setup_bucket with "unauthorized: access to the requested resource is not authorized" on every PR's test-julia jobs, unrelated to anything in this PR's own diff. Mirrors the same fix already applied in raicode's docker/iceberg compose stack: build minio and mc locally from MinIO's last published binary release (fetched from GitHub Releases, checksum-verified) instead of pulling a registry image. Verified locally: `docker compose up -d minio setup_bucket polaris init table-creation` brings up the full stack and table-creation completes successfully. Co-Authored-By: Claude Sonnet 5 --- docker/docker-compose.yml | 12 ++++++++++-- docker/mc/Dockerfile | 19 +++++++++++++++++++ docker/minio/Dockerfile | 27 +++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 docker/mc/Dockerfile create mode 100644 docker/minio/Dockerfile diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 4e25d79..6387b86 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -4,7 +4,13 @@ networks: services: minio: container_name: minio - image: quay.io/minio/minio:latest + # MinIO discontinued prebuilt community images (quay.io/minio/minio and + # minio/minio on Docker Hub are both unpullable now -- see + # docker/minio/Dockerfile). Built locally from their last binary + # release until this is mirrored into a registry we control -- `image:` + # here just names/tags the local build result for now. + build: ./minio + image: rustyiceberg/minio:RELEASE.2025-09-07T16-13-09Z ports: - "9000:9000" # MinIO API port - "9001:9001" # MinIO Console port @@ -24,7 +30,9 @@ services: retries: 5 setup_bucket: - image: quay.io/minio/mc:latest + # See the minio service above -- same situation for quay.io/minio/mc. + build: ./mc + image: rustyiceberg/mc:RELEASE.2025-08-13T08-35-41Z depends_on: minio: condition: service_healthy diff --git a/docker/mc/Dockerfile b/docker/mc/Dockerfile new file mode 100644 index 0000000..e7b4638 --- /dev/null +++ b/docker/mc/Dockerfile @@ -0,0 +1,19 @@ +# See docker/iceberg/minio/Dockerfile's header: quay.io/minio/mc and +# minio/mc (Docker Hub) are both unpullable now that MinIO has discontinued +# prebuilt community images. This builds the mc client from MinIO's last +# published binary release, fetched from GitHub Releases and checksum-verified. +FROM debian:bookworm-slim + +ARG MC_RELEASE=RELEASE.2025-08-13T08-35-41Z +ARG MC_SHA256=01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -fsSL -o /usr/bin/mc \ + "https://github.com/minio/mc/releases/download/${MC_RELEASE}/mc.linux-amd64.${MC_RELEASE}" \ + && echo "${MC_SHA256} /usr/bin/mc" | sha256sum -c - \ + && chmod +x /usr/bin/mc + +ENTRYPOINT ["mc"] diff --git a/docker/minio/Dockerfile b/docker/minio/Dockerfile new file mode 100644 index 0000000..b5a78ca --- /dev/null +++ b/docker/minio/Dockerfile @@ -0,0 +1,27 @@ +# MinIO discontinued prebuilt community images (see minio/minio's README as +# of Oct 2025: "THIS REPOSITORY IS NO LONGER MAINTAINED" / "distributed as +# source code only") -- quay.io/minio/minio and minio/minio (Docker Hub) are +# both unpullable now, and even upstream's own Dockerfile is broken (it FROMs +# minio/minio:latest, the very image that no longer exists). +# +# This builds a drop-in replacement from MinIO's last published binary +# release instead, fetched directly from GitHub Releases (still served, +# unlike dl.min.io which now returns 410) and checksum-verified. +FROM debian:bookworm-slim + +ARG MINIO_RELEASE=RELEASE.2025-09-07T16-13-09Z +ARG MINIO_SHA256=7c5bd8512c6e966455b1d198209358b2d191c77a83ab377c4073281065fb855f + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -fsSL -o /usr/bin/minio \ + "https://github.com/minio/minio/releases/download/${MINIO_RELEASE}/minio.linux-amd64.${MINIO_RELEASE}" \ + && echo "${MINIO_SHA256} /usr/bin/minio" | sha256sum -c - \ + && chmod +x /usr/bin/minio + +EXPOSE 9000 9001 +VOLUME ["/data"] +ENTRYPOINT ["minio"] +CMD ["server", "/data"]