diff --git a/Project.toml b/Project.toml index b8e0706..8534986 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "RustyIceberg" uuid = "390bdf5b-b624-43dc-a846-0ef7a3405804" -version = "0.9.2" +version = "0.9.3" [deps] Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45" diff --git a/iceberg_rust_ffi/Cargo.lock b/iceberg_rust_ffi/Cargo.lock index 2cfcbf6..2019a86 100644 --- a/iceberg_rust_ffi/Cargo.lock +++ b/iceberg_rust_ffi/Cargo.lock @@ -1786,7 +1786,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.10.0" -source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=1953bcee83b135848c69755b5e8eef101dcd155e#1953bcee83b135848c69755b5e8eef101dcd155e" +source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=9eb8921f941582b0165954dcd2e28ef3a4c3d56a#9eb8921f941582b0165954dcd2e28ef3a4c3d56a" dependencies = [ "aes-gcm", "anyhow", @@ -1849,7 +1849,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-rest" version = "0.10.0" -source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=1953bcee83b135848c69755b5e8eef101dcd155e#1953bcee83b135848c69755b5e8eef101dcd155e" +source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=9eb8921f941582b0165954dcd2e28ef3a4c3d56a#9eb8921f941582b0165954dcd2e28ef3a4c3d56a" dependencies = [ "async-trait", "chrono", @@ -1869,7 +1869,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" version = "0.10.0" -source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=1953bcee83b135848c69755b5e8eef101dcd155e#1953bcee83b135848c69755b5e8eef101dcd155e" +source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=9eb8921f941582b0165954dcd2e28ef3a4c3d56a#9eb8921f941582b0165954dcd2e28ef3a4c3d56a" dependencies = [ "proc-macro2", "quote", @@ -1878,7 +1878,7 @@ dependencies = [ [[package]] name = "iceberg_rust_ffi" -version = "0.9.2" +version = "0.9.3" dependencies = [ "anyhow", "arrow-array", @@ -2712,6 +2712,7 @@ dependencies = [ "opendal-layer-timeout", "opendal-service-azdls", "opendal-service-fs", + "opendal-service-http", "opendal-service-s3", ] @@ -2843,6 +2844,18 @@ dependencies = [ "xattr", ] +[[package]] +name = "opendal-service-http" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2ebcbdfad4b81a1e6dde70e898396e011dfd52b5f8639b2628656f7915d6f4c" +dependencies = [ + "http", + "log", + "opendal-core", + "serde", +] + [[package]] name = "opendal-service-s3" version = "0.58.2" diff --git a/iceberg_rust_ffi/Cargo.toml b/iceberg_rust_ffi/Cargo.toml index 23089a9..5c26cbb 100644 --- a/iceberg_rust_ffi/Cargo.toml +++ b/iceberg_rust_ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iceberg_rust_ffi" -version = "0.9.2" +version = "0.9.3" edition = "2021" [lib] @@ -12,8 +12,8 @@ default = ["julia"] julia = [] [dependencies] -iceberg = { git = "https://github.com/RelationalAI/iceberg-rust.git", rev = "1953bcee83b135848c69755b5e8eef101dcd155e", features = ["storage-azdls"] } -iceberg-catalog-rest = { git = "https://github.com/RelationalAI/iceberg-rust.git", rev = "1953bcee83b135848c69755b5e8eef101dcd155e" } +iceberg = { git = "https://github.com/RelationalAI/iceberg-rust.git", rev = "9eb8921f941582b0165954dcd2e28ef3a4c3d56a", features = ["storage-azdls"] } +iceberg-catalog-rest = { git = "https://github.com/RelationalAI/iceberg-rust.git", rev = "9eb8921f941582b0165954dcd2e28ef3a4c3d56a" } object_store_ffi = { git = "https://github.com/RelationalAI/object_store_ffi", rev = "bbbbd2f0ab0d609416014cef0d124efa99a515f3", default-features = false } tokio = { version = "1.0", features = ["full"] } futures = "0.3" diff --git a/test/error_tests.jl b/test/error_tests.jl index 35b2669..7751ca5 100644 --- a/test/error_tests.jl +++ b/test/error_tests.jl @@ -155,6 +155,115 @@ end end end + @testset "s3.allow-anonymous unconditionally skips signing, even with valid credentials" begin + # Regression coverage for a real production incident (raicode's Iceberg + # data-loading integration always sends "s3.allow-anonymous" => "true" + # alongside any explicit credentials, intending it as an anonymous + # *fallback* for when no credentials are configured). + # + # Under the fork's old opendal (0.55), that was safe: `allow_anonymous` + # was only consulted *after* trying to load a real credential, and only + # skipped signing if none was found (opendal-0.55.0/src/services/s3/core.rs, + # `S3Core::load_credential`). + # + # Upstream's opendal 0.58 (pulled in by this repo's Aug 2026 iceberg-rust + # merge, RelationalAI/iceberg-rust#78) renamed the flag to `skip_signature` + # and changed its semantics: it's now checked *first*, unconditionally, + # before any credential is even loaded (opendal-service-s3-0.58.2/src/ + # core.rs: `if self.skip_signature { return ...unsigned... }`). So valid + # credentials configured alongside "s3.allow-anonymous" => "true" are now + # silently ignored — every request goes out unsigned. + # + # Against a private bucket (as production tables are), this surfaces as a + # 403 AccessDenied / AUTH_FAILED instead of a successful read, since + # anonymous requests aren't authorized. Callers must omit + # "s3.allow-anonymous" whenever credentials are being provided; do not set + # it "just in case" alongside real credentials. + without_aws_env() do + s3 = get_s3_config() + nation_meta = "s3://warehouse/tpch.sf01/nation/metadata/" * + "00001-44f668fe-3688-49d5-851f-36e75d143321.metadata.json" + base_properties = Dict( + "s3.endpoint" => s3["endpoint"], + "s3.access-key-id" => s3["access_key_id"], + "s3.secret-access-key" => s3["secret_access_key"], + "s3.region" => s3["region"], + "s3.path-style-access" => "true", + ) + + @testset "allow-anonymous set alongside valid credentials -> credentials ignored, AUTH_FAILED" begin + exc = try + table_open( + nation_meta; + properties=merge(base_properties, Dict("s3.allow-anonymous" => "true")) + ) + nothing + catch e + e isa RustyIceberg.IcebergException ? e : rethrow() + end + @test !isnothing(exc) + @test exc.code == RustyIceberg.AUTH_FAILED + end + + @testset "allow-anonymous omitted with the same valid credentials -> succeeds" begin + table = table_open(nation_meta; properties=base_properties) + @test table != C_NULL + free_table(table) + end + end + end + + @testset "adls.allow-anonymous enables reading a public Azure container" begin + # Regression coverage for the ADLS counterpart of the S3 + # allow-anonymous incident above: unlike S3 (which exposes an + # explicit "skip signing" bypass), opendal's Azdls service had no + # anonymous-access mode at all -- every request went through a + # generic credential-provider chain whose Credential variants all + # require a non-empty secret to be considered valid, and hard-errors + # ("failed to load signing credential" / AUTH_FAILED) if none can be + # found or validated. An empty SAS token doesn't help either: it's + # treated as a *found but invalid* credential, which fails the same + # way. So reading a genuinely public, unauthenticated Azure + # container was not possible through this stack at all until + # RelationalAI/iceberg-rust#79 added an explicit + # "adls.allow-anonymous" opt-in (mirroring "s3.allow-anonymous"), + # scoped to opendal's generic Http service instead of the Azdls one. + # + # This surfaced as 4 failing tests/DataLoader/iceberg_tests.jl "load + # from azure" (public access) cases in raicode while validating the + # RustyIceberg 0.9.2 bump (PR #28144) -- opendal was bumped 0.55->0.58 + # as part of that merge, and this gap came along for the ride (the + # old fork's opendal 0.55 had the exact same gap; it just happened + # not to be exercised until then). + # + # Uses the same public-access test table raicode's own integration + # tests read from. + without_azure_env() do + nation_meta = "abfss://integration-tests@publicbenchmarks.dfs.core.windows.net" * + "/iceberg/nation/metadata/00001-44f668fe-3688-49d5-851f-36e75d143321.metadata.json" + + @testset "no credentials, no allow-anonymous -> AUTH_FAILED" begin + exc = try + table_open(nation_meta) + nothing + catch e + e isa RustyIceberg.IcebergException ? e : rethrow() + end + @test !isnothing(exc) + @test exc.code == RustyIceberg.AUTH_FAILED + end + + @testset "adls.allow-anonymous=true -> succeeds" begin + table = table_open( + nation_meta; + properties=Dict("adls.allow-anonymous" => "true"), + ) + @test table != C_NULL + free_table(table) + end + end + end + end # ── Scan — missing / corrupted Parquet files ────────────────────────────────── diff --git a/test/test_config.jl b/test/test_config.jl index 09a3377..7a6cf39 100644 --- a/test/test_config.jl +++ b/test/test_config.jl @@ -132,3 +132,35 @@ function without_aws_env(f) end end end + +const AZURE_ENV_VARS = ["AZBLOB_ACCOUNT_NAME", "AZURE_STORAGE_ACCOUNT_NAME", + "AZBLOB_ACCOUNT_KEY", "AZURE_STORAGE_ACCOUNT_KEY", + "AZURE_STORAGE_SAS_TOKEN", "AZURE_STORAGE_BEARER_TOKEN", + "AZURE_CLIENT_ID", "AZURE_CLIENT_SECRET", "AZURE_TENANT_ID", + "AZURE_AUTHORITY_HOST"] + +""" + without_azure_env(f) + +Run `f()` with all ambient Azure credential environment variables unset, then +restore them. Useful for tests that need to verify anonymous/unsigned access +actually happens, rather than accidentally succeeding via an ambient +credential (managed identity, Azure CLI, environment variables) picked up by +opendal's default Azure credential chain. +""" +function without_azure_env(f) + saved = Dict{String,String}() + for var in AZURE_ENV_VARS + if haskey(ENV, var) + saved[var] = ENV[var] + delete!(ENV, var) + end + end + try + f() + finally + for (var, val) in saved + ENV[var] = val + end + end +end