From c3bc0c2e5b15c42aed94409cdefecebe9b269e84 Mon Sep 17 00:00:00 2001 From: Gerald Berger Date: Wed, 2 Sep 2026 09:36:13 +0200 Subject: [PATCH 1/3] test: pin s3.allow-anonymous's credential-ignoring behavior under opendal 0.58 Regression coverage for a real production incident: raicode's Iceberg data-loading integration always sends "s3.allow-anonymous" => "true" alongside any explicit credentials, intending it purely as a fallback for when no credentials are configured. Under the fork's old opendal (0.55), that was safe -- allow_anonymous was only consulted after trying to load a real credential, and only skipped signing if none was found. Upstream's opendal 0.58 (pulled in by the Aug 2026 iceberg-rust merge, RelationalAI/iceberg-rust#78) renamed the flag to skip_signature and changed its semantics: it's now checked first, unconditionally, before any credential is even loaded. Configured credentials are silently ignored whenever allow-anonymous is also set, so every request goes out unsigned -- which a private bucket correctly rejects with 403/AUTH_FAILED. Verified locally against this repo's own docker-compose MinIO/Polaris setup: both assertions pass against the current (0.9.2) FFI build. Co-Authored-By: Claude Sonnet 5 --- test/error_tests.jl | 58 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/test/error_tests.jl b/test/error_tests.jl index 35b2669..ef4f61a 100644 --- a/test/error_tests.jl +++ b/test/error_tests.jl @@ -155,6 +155,64 @@ end end end + @testset "s3.allow-anonymous unconditionally skips signing, even with valid credentials" begin + # Regression coverage for a real production incident (raicode's Iceberg + # data-loading integration always sends "s3.allow-anonymous" => "true" + # alongside any explicit credentials, intending it as an anonymous + # *fallback* for when no credentials are configured). + # + # Under the fork's old opendal (0.55), that was safe: `allow_anonymous` + # was only consulted *after* trying to load a real credential, and only + # skipped signing if none was found (opendal-0.55.0/src/services/s3/core.rs, + # `S3Core::load_credential`). + # + # Upstream's opendal 0.58 (pulled in by this repo's Aug 2026 iceberg-rust + # merge, RelationalAI/iceberg-rust#78) renamed the flag to `skip_signature` + # and changed its semantics: it's now checked *first*, unconditionally, + # before any credential is even loaded (opendal-service-s3-0.58.2/src/ + # core.rs: `if self.skip_signature { return ...unsigned... }`). So valid + # credentials configured alongside "s3.allow-anonymous" => "true" are now + # silently ignored — every request goes out unsigned. + # + # Against a private bucket (as production tables are), this surfaces as a + # 403 AccessDenied / AUTH_FAILED instead of a successful read, since + # anonymous requests aren't authorized. Callers must omit + # "s3.allow-anonymous" whenever credentials are being provided; do not set + # it "just in case" alongside real credentials. + without_aws_env() do + s3 = get_s3_config() + nation_meta = "s3://warehouse/tpch.sf01/nation/metadata/" * + "00001-44f668fe-3688-49d5-851f-36e75d143321.metadata.json" + base_properties = Dict( + "s3.endpoint" => s3["endpoint"], + "s3.access-key-id" => s3["access_key_id"], + "s3.secret-access-key" => s3["secret_access_key"], + "s3.region" => s3["region"], + "s3.path-style-access" => "true", + ) + + @testset "allow-anonymous set alongside valid credentials -> credentials ignored, AUTH_FAILED" begin + exc = try + table_open( + nation_meta; + properties=merge(base_properties, Dict("s3.allow-anonymous" => "true")) + ) + nothing + catch e + e isa RustyIceberg.IcebergException ? e : rethrow() + end + @test !isnothing(exc) + @test exc.code == RustyIceberg.AUTH_FAILED + end + + @testset "allow-anonymous omitted with the same valid credentials -> succeeds" begin + table = table_open(nation_meta; properties=base_properties) + @test table != C_NULL + free_table(table) + end + end + end + end # ── Scan — missing / corrupted Parquet files ────────────────────────────────── From 696b62f0e058746d130a534debee8dad17e842af Mon Sep 17 00:00:00 2001 From: Gerald Berger Date: Wed, 2 Sep 2026 13:51:37 +0200 Subject: [PATCH 2/3] test: add reproduction for adls.allow-anonymous (opendal 0.58 ADLS gap) Regression coverage for the ADLS counterpart of the s3.allow-anonymous incident already covered in this PR: unlike S3's opendal service (which exposes an explicit skip_signature bypass), opendal's Azdls service has no anonymous/unsigned mode at all -- every request goes through a generic credential-provider chain that hard errors ("failed to load signing credential") if no credential can be found or validated. An empty adls.sas-token doesn't help either: it's treated as a found-but- invalid credential, which fails the same way. So reading a genuinely public, unauthenticated Azure container was not possible through this stack at all. RelationalAI/iceberg-rust#79 adds an explicit "adls.allow-anonymous" opt-in (mirroring s3.allow-anonymous), routing to opendal's generic Http service instead of the signed Azdls one. Verified locally against this repo's own docker-compose infra, plus the real public Azure test container (publicbenchmarks.dfs.core.windows.net/integration-tests, the same one RelationalAI/raicode's own integration tests read from): without the flag, table_open correctly fails with AUTH_FAILED; with it, the read succeeds. Bumps the git rev to iceberg-rust#79's branch tip for this verification; needs a follow-up bump to the actual merge commit once that PR lands (flagged with a TODO in Cargo.toml). Version bumped to 0.9.3 to match. Co-Authored-By: Claude Sonnet 5 --- Project.toml | 2 +- iceberg_rust_ffi/Cargo.lock | 21 ++++++++++++--- iceberg_rust_ffi/Cargo.toml | 10 +++++--- test/error_tests.jl | 51 +++++++++++++++++++++++++++++++++++++ test/test_config.jl | 32 +++++++++++++++++++++++ 5 files changed, 108 insertions(+), 8 deletions(-) diff --git a/Project.toml b/Project.toml index b8e0706..8534986 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "RustyIceberg" uuid = "390bdf5b-b624-43dc-a846-0ef7a3405804" -version = "0.9.2" +version = "0.9.3" [deps] Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45" diff --git a/iceberg_rust_ffi/Cargo.lock b/iceberg_rust_ffi/Cargo.lock index 2cfcbf6..6cf8302 100644 --- a/iceberg_rust_ffi/Cargo.lock +++ b/iceberg_rust_ffi/Cargo.lock @@ -1786,7 +1786,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.10.0" -source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=1953bcee83b135848c69755b5e8eef101dcd155e#1953bcee83b135848c69755b5e8eef101dcd155e" +source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=8d2a2a3aa6bd7f4200a4fb61bfe7124a3434b8bc#8d2a2a3aa6bd7f4200a4fb61bfe7124a3434b8bc" dependencies = [ "aes-gcm", "anyhow", @@ -1849,7 +1849,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-rest" version = "0.10.0" -source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=1953bcee83b135848c69755b5e8eef101dcd155e#1953bcee83b135848c69755b5e8eef101dcd155e" +source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=8d2a2a3aa6bd7f4200a4fb61bfe7124a3434b8bc#8d2a2a3aa6bd7f4200a4fb61bfe7124a3434b8bc" dependencies = [ "async-trait", "chrono", @@ -1869,7 +1869,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" version = "0.10.0" -source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=1953bcee83b135848c69755b5e8eef101dcd155e#1953bcee83b135848c69755b5e8eef101dcd155e" +source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=8d2a2a3aa6bd7f4200a4fb61bfe7124a3434b8bc#8d2a2a3aa6bd7f4200a4fb61bfe7124a3434b8bc" dependencies = [ "proc-macro2", "quote", @@ -1878,7 +1878,7 @@ dependencies = [ [[package]] name = "iceberg_rust_ffi" -version = "0.9.2" +version = "0.9.3" dependencies = [ "anyhow", "arrow-array", @@ -2712,6 +2712,7 @@ dependencies = [ "opendal-layer-timeout", "opendal-service-azdls", "opendal-service-fs", + "opendal-service-http", "opendal-service-s3", ] @@ -2843,6 +2844,18 @@ dependencies = [ "xattr", ] +[[package]] +name = "opendal-service-http" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2ebcbdfad4b81a1e6dde70e898396e011dfd52b5f8639b2628656f7915d6f4c" +dependencies = [ + "http", + "log", + "opendal-core", + "serde", +] + [[package]] name = "opendal-service-s3" version = "0.58.2" diff --git a/iceberg_rust_ffi/Cargo.toml b/iceberg_rust_ffi/Cargo.toml index 23089a9..7a44ed2 100644 --- a/iceberg_rust_ffi/Cargo.toml +++ b/iceberg_rust_ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iceberg_rust_ffi" -version = "0.9.2" +version = "0.9.3" edition = "2021" [lib] @@ -12,8 +12,12 @@ default = ["julia"] julia = [] [dependencies] -iceberg = { git = "https://github.com/RelationalAI/iceberg-rust.git", rev = "1953bcee83b135848c69755b5e8eef101dcd155e", features = ["storage-azdls"] } -iceberg-catalog-rest = { git = "https://github.com/RelationalAI/iceberg-rust.git", rev = "1953bcee83b135848c69755b5e8eef101dcd155e" } +# TODO: rev points at the unmerged gb/adls-anonymous-access branch +# (RelationalAI/iceberg-rust#79), for local verification of the +# adls.allow-anonymous fix this PR's new test exercises. Bump to the actual +# merge commit on main once #79 lands. +iceberg = { git = "https://github.com/RelationalAI/iceberg-rust.git", rev = "8d2a2a3aa6bd7f4200a4fb61bfe7124a3434b8bc", features = ["storage-azdls"] } +iceberg-catalog-rest = { git = "https://github.com/RelationalAI/iceberg-rust.git", rev = "8d2a2a3aa6bd7f4200a4fb61bfe7124a3434b8bc" } object_store_ffi = { git = "https://github.com/RelationalAI/object_store_ffi", rev = "bbbbd2f0ab0d609416014cef0d124efa99a515f3", default-features = false } tokio = { version = "1.0", features = ["full"] } futures = "0.3" diff --git a/test/error_tests.jl b/test/error_tests.jl index ef4f61a..7751ca5 100644 --- a/test/error_tests.jl +++ b/test/error_tests.jl @@ -213,6 +213,57 @@ end end end + @testset "adls.allow-anonymous enables reading a public Azure container" begin + # Regression coverage for the ADLS counterpart of the S3 + # allow-anonymous incident above: unlike S3 (which exposes an + # explicit "skip signing" bypass), opendal's Azdls service had no + # anonymous-access mode at all -- every request went through a + # generic credential-provider chain whose Credential variants all + # require a non-empty secret to be considered valid, and hard-errors + # ("failed to load signing credential" / AUTH_FAILED) if none can be + # found or validated. An empty SAS token doesn't help either: it's + # treated as a *found but invalid* credential, which fails the same + # way. So reading a genuinely public, unauthenticated Azure + # container was not possible through this stack at all until + # RelationalAI/iceberg-rust#79 added an explicit + # "adls.allow-anonymous" opt-in (mirroring "s3.allow-anonymous"), + # scoped to opendal's generic Http service instead of the Azdls one. + # + # This surfaced as 4 failing tests/DataLoader/iceberg_tests.jl "load + # from azure" (public access) cases in raicode while validating the + # RustyIceberg 0.9.2 bump (PR #28144) -- opendal was bumped 0.55->0.58 + # as part of that merge, and this gap came along for the ride (the + # old fork's opendal 0.55 had the exact same gap; it just happened + # not to be exercised until then). + # + # Uses the same public-access test table raicode's own integration + # tests read from. + without_azure_env() do + nation_meta = "abfss://integration-tests@publicbenchmarks.dfs.core.windows.net" * + "/iceberg/nation/metadata/00001-44f668fe-3688-49d5-851f-36e75d143321.metadata.json" + + @testset "no credentials, no allow-anonymous -> AUTH_FAILED" begin + exc = try + table_open(nation_meta) + nothing + catch e + e isa RustyIceberg.IcebergException ? e : rethrow() + end + @test !isnothing(exc) + @test exc.code == RustyIceberg.AUTH_FAILED + end + + @testset "adls.allow-anonymous=true -> succeeds" begin + table = table_open( + nation_meta; + properties=Dict("adls.allow-anonymous" => "true"), + ) + @test table != C_NULL + free_table(table) + end + end + end + end # ── Scan — missing / corrupted Parquet files ────────────────────────────────── diff --git a/test/test_config.jl b/test/test_config.jl index 09a3377..7a6cf39 100644 --- a/test/test_config.jl +++ b/test/test_config.jl @@ -132,3 +132,35 @@ function without_aws_env(f) end end end + +const AZURE_ENV_VARS = ["AZBLOB_ACCOUNT_NAME", "AZURE_STORAGE_ACCOUNT_NAME", + "AZBLOB_ACCOUNT_KEY", "AZURE_STORAGE_ACCOUNT_KEY", + "AZURE_STORAGE_SAS_TOKEN", "AZURE_STORAGE_BEARER_TOKEN", + "AZURE_CLIENT_ID", "AZURE_CLIENT_SECRET", "AZURE_TENANT_ID", + "AZURE_AUTHORITY_HOST"] + +""" + without_azure_env(f) + +Run `f()` with all ambient Azure credential environment variables unset, then +restore them. Useful for tests that need to verify anonymous/unsigned access +actually happens, rather than accidentally succeeding via an ambient +credential (managed identity, Azure CLI, environment variables) picked up by +opendal's default Azure credential chain. +""" +function without_azure_env(f) + saved = Dict{String,String}() + for var in AZURE_ENV_VARS + if haskey(ENV, var) + saved[var] = ENV[var] + delete!(ENV, var) + end + end + try + f() + finally + for (var, val) in saved + ENV[var] = val + end + end +end From 07cd5bc6f9e79697fa684fc93ad43470ce60c8a5 Mon Sep 17 00:00:00 2001 From: Gerald Berger Date: Wed, 2 Sep 2026 14:00:12 +0200 Subject: [PATCH 3/3] Point iceberg-rust dep at merged adls.allow-anonymous commit PR #79 (RelationalAI/iceberg-rust) merged as 9eb8921f9; switch from the temporary WIP branch-tip rev to the real main commit. --- iceberg_rust_ffi/Cargo.lock | 6 +++--- iceberg_rust_ffi/Cargo.toml | 8 ++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/iceberg_rust_ffi/Cargo.lock b/iceberg_rust_ffi/Cargo.lock index 6cf8302..2019a86 100644 --- a/iceberg_rust_ffi/Cargo.lock +++ b/iceberg_rust_ffi/Cargo.lock @@ -1786,7 +1786,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.10.0" -source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=8d2a2a3aa6bd7f4200a4fb61bfe7124a3434b8bc#8d2a2a3aa6bd7f4200a4fb61bfe7124a3434b8bc" +source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=9eb8921f941582b0165954dcd2e28ef3a4c3d56a#9eb8921f941582b0165954dcd2e28ef3a4c3d56a" dependencies = [ "aes-gcm", "anyhow", @@ -1849,7 +1849,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-rest" version = "0.10.0" -source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=8d2a2a3aa6bd7f4200a4fb61bfe7124a3434b8bc#8d2a2a3aa6bd7f4200a4fb61bfe7124a3434b8bc" +source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=9eb8921f941582b0165954dcd2e28ef3a4c3d56a#9eb8921f941582b0165954dcd2e28ef3a4c3d56a" dependencies = [ "async-trait", "chrono", @@ -1869,7 +1869,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" version = "0.10.0" -source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=8d2a2a3aa6bd7f4200a4fb61bfe7124a3434b8bc#8d2a2a3aa6bd7f4200a4fb61bfe7124a3434b8bc" +source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=9eb8921f941582b0165954dcd2e28ef3a4c3d56a#9eb8921f941582b0165954dcd2e28ef3a4c3d56a" dependencies = [ "proc-macro2", "quote", diff --git a/iceberg_rust_ffi/Cargo.toml b/iceberg_rust_ffi/Cargo.toml index 7a44ed2..5c26cbb 100644 --- a/iceberg_rust_ffi/Cargo.toml +++ b/iceberg_rust_ffi/Cargo.toml @@ -12,12 +12,8 @@ default = ["julia"] julia = [] [dependencies] -# TODO: rev points at the unmerged gb/adls-anonymous-access branch -# (RelationalAI/iceberg-rust#79), for local verification of the -# adls.allow-anonymous fix this PR's new test exercises. Bump to the actual -# merge commit on main once #79 lands. -iceberg = { git = "https://github.com/RelationalAI/iceberg-rust.git", rev = "8d2a2a3aa6bd7f4200a4fb61bfe7124a3434b8bc", features = ["storage-azdls"] } -iceberg-catalog-rest = { git = "https://github.com/RelationalAI/iceberg-rust.git", rev = "8d2a2a3aa6bd7f4200a4fb61bfe7124a3434b8bc" } +iceberg = { git = "https://github.com/RelationalAI/iceberg-rust.git", rev = "9eb8921f941582b0165954dcd2e28ef3a4c3d56a", features = ["storage-azdls"] } +iceberg-catalog-rest = { git = "https://github.com/RelationalAI/iceberg-rust.git", rev = "9eb8921f941582b0165954dcd2e28ef3a4c3d56a" } object_store_ffi = { git = "https://github.com/RelationalAI/object_store_ffi", rev = "bbbbd2f0ab0d609416014cef0d124efa99a515f3", default-features = false } tokio = { version = "1.0", features = ["full"] } futures = "0.3"