diff --git a/Cargo.lock b/Cargo.lock index 2f0999cb2..321544845 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,6 +80,17 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "alp" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b23203d5e87df8d8393c8021b53f9193b697ba13fc4ef653911dc1235e87bfd7" +dependencies = [ + "fastlanes 0.6.1", + "itertools 0.15.0", + "num-traits", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -159,7 +170,7 @@ dependencies = [ "liblzma", "log", "miniz_oxide", - "num-bigint", + "num-bigint 0.4.8", "quad-rand", "rand 0.9.5", "regex-lite", @@ -221,9 +232,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" +checksum = "3f04e0a89cc175eaf3740282537851e6fd9d7276f97730b9d0ea2859c11a85df" dependencies = [ "arrow-arith", "arrow-array", @@ -243,9 +254,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +checksum = "157943c720919e8f5b6731407ce5cd4c9a5ef47a75bf10edaf24b7081b5763ef" dependencies = [ "arrow-array", "arrow-buffer", @@ -257,9 +268,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" +checksum = "72b5b1682924f11ecf6929689c16cbc69efbb6dd141b06ad98ee1cb10d858c43" dependencies = [ "ahash", "arrow-buffer", @@ -269,6 +280,7 @@ dependencies = [ "chrono-tz", "half", "hashbrown 0.17.1", + "libc", "num-complex", "num-integer", "num-traits", @@ -276,21 +288,21 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" +checksum = "b5d687afc6c927965a8a759f3b68de91933d9dfa2e1bb6e156bac84b85665e92" dependencies = [ "bytes", "half", - "num-bigint", + "num-bigint 0.5.1", "num-traits", ] [[package]] name = "arrow-cast" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" +checksum = "3f23c3379feede42135ef27ca60e67f6e643cd4343ae74e4c8b16c83f55afd85" dependencies = [ "arrow-array", "arrow-buffer", @@ -299,7 +311,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64 0.22.1", + "base64 0.23.1", "chrono", "comfy-table", "half", @@ -310,9 +322,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26" +checksum = "85b1984628b091b8319b3c305fce21d04f9e9b2578bb733b0d0108462ae1d698" dependencies = [ "arrow-array", "arrow-cast", @@ -325,9 +337,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" +checksum = "bb0a031c7ff5e2bdb443117054fb466af1d4df5b7f0ecbe197713dc8b113b1ba" dependencies = [ "arrow-buffer", "arrow-schema", @@ -338,9 +350,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" +checksum = "f5dd1afadda2eb821aaf851bb4c0adef60c30cc0f1a29bd02fb99dd07fb9f58d" dependencies = [ "arrow-array", "arrow-buffer", @@ -348,15 +360,15 @@ dependencies = [ "arrow-schema", "arrow-select", "flatbuffers", - "lz4_flex 0.13.1", + "lz4_flex 0.14.0", "zstd 0.13.3", ] [[package]] name = "arrow-json" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14" +checksum = "e2bc4d56a0dad84cc51c5812ccff48b642047a1f3be80349cd84318983e79edb" dependencies = [ "arrow-array", "arrow-buffer", @@ -379,9 +391,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +checksum = "561b8209faa2cd489764b7fd07d7cb9e8d274d78048c38de59f77c41140cb834" dependencies = [ "arrow-array", "arrow-buffer", @@ -392,9 +404,9 @@ dependencies = [ [[package]] name = "arrow-pyarrow" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ffb9be5a873590f825aef50df20e0f8dff5fd42a77058e28bbe7bd44bb53dec" +checksum = "ba037102b3f52163f96db970209ada931c2a6f46c4607d3170950d075db68773" dependencies = [ "arrow-array", "arrow-data", @@ -404,9 +416,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" +checksum = "0d8c68b87a569f92b90c46cb0a09d86bd9609ddc9b1bf1fc5a503c8c6907c14e" dependencies = [ "arrow-array", "arrow-buffer", @@ -417,9 +429,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" +checksum = "e87acb0c6bf6237fca7a50d7b51e801c98a18a5fe6f3bb34f7d77b85ca165388" dependencies = [ "bitflags 2.13.1", "serde_core", @@ -428,9 +440,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +checksum = "c844141958d805b47d1208e0d12aac53fcf388ce8011639fe82aa492adde0bb4" dependencies = [ "ahash", "arrow-array", @@ -442,9 +454,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" +checksum = "bc130892b1a9552da77971924b390e06cf375d1f7e4344a7b7b40ca4426fa470" dependencies = [ "arrow-array", "arrow-buffer", @@ -624,7 +636,7 @@ checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.9", ] [[package]] @@ -787,7 +799,7 @@ checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", - "num-bigint", + "num-bigint 0.4.8", "num-integer", "num-traits", "serde", @@ -813,10 +825,11 @@ dependencies = [ [[package]] name = "bit-vec" -version = "0.9.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269" dependencies = [ + "borsh", "serde", ] @@ -950,6 +963,30 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "brotli" version = "8.0.4" @@ -1333,12 +1370,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core_detect" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" - [[package]] name = "cpubits" version = "0.1.1" @@ -1563,9 +1594,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" +checksum = "4cd5317db77ba6bca25c761a8091b9d294a9f4188e4f9dcd0243e6a57de862ac" dependencies = [ "arrow", "arrow-schema", @@ -1600,7 +1631,7 @@ dependencies = [ "flate2", "futures", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -1616,9 +1647,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" +checksum = "f35d36794509b7bfcbf90e11a1b5e4d047123c34610105e2e80a2048351cdb2f" dependencies = [ "arrow", "async-trait", @@ -1632,7 +1663,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -1641,9 +1672,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" +checksum = "a6454bfb7f43b83942e2ca2576fd09342eedbd0d323e1eef72c3c8e4d57d7203" dependencies = [ "arrow", "async-trait", @@ -1657,16 +1688,17 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", + "percent-encoding", ] [[package]] name = "datafusion-common" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" +checksum = "d26626e12b224b83351239465dcd00f72d47b02fb5e5b65a643e2dc8ed37f687" dependencies = [ "arrow", "arrow-ipc", @@ -1676,9 +1708,10 @@ dependencies = [ "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "libc", "log", + "num-traits", "object_store", "parquet", "recursive", @@ -1690,9 +1723,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" +checksum = "a106420cbfa980f242c22243ae9a6cce661d4d86ce96cff87dacabf7754a4470" dependencies = [ "futures", "log", @@ -1701,9 +1734,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" +checksum = "210aa1e9525e9454f2399dbb8b0b95510145bd8c1fdc35de74e9ccbf81bcb699" dependencies = [ "arrow", "async-compression", @@ -1719,11 +1752,12 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "flate2", "futures", "glob", - "itertools 0.14.0", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -1737,9 +1771,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" +checksum = "65b1b041688cad707fa103c3e7f4c6d89b3134ebf67da0129070dad3381851eb" dependencies = [ "arrow", "arrow-ipc", @@ -1752,18 +1786,19 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "object_store", "tokio", ] [[package]] name = "datafusion-datasource-csv" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" +checksum = "2a7b34e253c4fa5251f174977af910a2f2ec831ec39dbbb56b29ce505cff278b" dependencies = [ "arrow", "async-trait", @@ -1775,6 +1810,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", "object_store", @@ -1784,9 +1820,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" +checksum = "1663338f43be6a5746473008e2d0fbde282577ddf0bae90d6c7386e988d71b29" dependencies = [ "arrow", "async-trait", @@ -1798,6 +1834,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", "object_store", @@ -1807,11 +1844,12 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" +checksum = "7c521ae91b6d899d7190812881f45fe9463c222c11d90149f5332a2e56555904" dependencies = [ "arrow", + "arrow-schema", "async-trait", "bytes", "datafusion-common", @@ -1825,10 +1863,11 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-pruning", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -1838,19 +1877,20 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" +checksum = "0c2569c2e74a34fcae3a369ab3912d61e41a062592510f11e31b1dae2725ecdc" [[package]] name = "datafusion-execution" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" +checksum = "39cfdd5ac0769d1e74a885450e49fde8f63ca7ab41eb479a068564aaea80b837" dependencies = [ "arrow", "arrow-buffer", "async-trait", + "bytes", "dashmap", "datafusion-common", "datafusion-expr", @@ -1859,16 +1899,19 @@ dependencies = [ "log", "object_store", "parking_lot", + "pin-project-lite", "rand 0.9.5", "tempfile", + "tokio", + "tokio-util", "url", ] [[package]] name = "datafusion-expr" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" +checksum = "86560fd7df4dda924f5474b588c56e7ae8e96dcf158c7327c9cfe9daff448ec5" dependencies = [ "arrow", "arrow-schema", @@ -1880,30 +1923,33 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "recursive", + "serde", "serde_json", "sqlparser", ] [[package]] name = "datafusion-expr-common" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" +checksum = "e82e313cf655cf6bc2dce6cbd23bcb0d20236a54a1b09c436615e4bac4d8c49f" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-ffi" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5660e8fa79fd51e29ce46f3026b67317ef738ebd633e106beb1a1907a406152" +checksum = "58a2da6b361eae9452cf8e2cafadb05b44501a4d965b26760327cb18e0d26c4f" dependencies = [ "arrow", "arrow-schema", @@ -1934,13 +1980,13 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" +checksum = "939ae2771da8a3fdc23ab4de610ae8fef9515476626b49d96d0d5c782b105771" dependencies = [ "arrow", "arrow-buffer", - "base64 0.22.1", + "base64 0.23.1", "blake2", "blake3", "chrono", @@ -1953,7 +1999,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hex", - "itertools 0.14.0", + "itertools 0.15.0", "log", "md-5 0.11.0", "memchr", @@ -1966,9 +2012,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" +checksum = "fd6c3aee9f2bc65716787d651e023743a90b4efbc57a2f679f036632e16fb110" dependencies = [ "arrow", "datafusion-common", @@ -1979,17 +2025,17 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", - "foldhash 0.2.0", "half", + "hashbrown 0.17.1", "log", "num-traits", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" +checksum = "988b5d3dfb4939d81c2f9fab267063610fe52a843620f2fa6cb2a1598e43369c" dependencies = [ "arrow", "datafusion-common", @@ -1999,9 +2045,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" +checksum = "08149d433a97c9ea3cdf80de06480ab5c7f5453485f1e452d5b3583c81d5c0dd" dependencies = [ "arrow", "arrow-ord", @@ -2016,7 +2062,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hashbrown 0.17.1", - "itertools 0.14.0", + "itertools 0.15.0", "itoa", "log", "memchr", @@ -2024,9 +2070,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" +checksum = "2c767567cce2782d4d0bdfa55fab010aaba0bdbe533509e9e0b9a6a3ee29ed59" dependencies = [ "arrow", "async-trait", @@ -2040,9 +2086,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" +checksum = "0a311c1f558d1f792f1f631c3670da12ee72a20fbf5d6ca151325d66191c03c1" dependencies = [ "arrow", "datafusion-common", @@ -2057,9 +2103,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" +checksum = "fffd54b8acf6ecef284cc0677e01e1b8c71be64fec50612ba00a0ff0d9b41a52" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2067,20 +2113,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" +checksum = "14d6df9763c8fb968b2417d797aef4c55ac2e4a895aaef38c72c16a287aa4a40" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] name = "datafusion-optimizer" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" +checksum = "acfb8cea6e05e2e75bae183340cf7d53cccb3db0449e2f1fbc07654d980fd19f" dependencies = [ "arrow", "chrono", @@ -2089,7 +2135,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-physical-expr", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "recursive", "regex", @@ -2098,20 +2144,22 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" +checksum = "9f002cef849e10c0597a217ec7891c2796c0f8b42d739f5d24abc24496373406" dependencies = [ "arrow", + "arrow-schema", "datafusion-common", "datafusion-expr", "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", + "datafusion-proto-models", "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "petgraph", "recursive", @@ -2120,9 +2168,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" +checksum = "82761522c4536e7059d3b185cd99f10525c2bf48f1bd26a0a57e73e19eadc0fb" dependencies = [ "arrow", "datafusion-common", @@ -2130,31 +2178,32 @@ dependencies = [ "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-expr-common", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-expr-common" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" +checksum = "f8c5c67aa252886f4d6b331c8114797f5b140464e774c4da36889269a1bf1bd8" dependencies = [ "arrow", "chrono", "datafusion-common", "datafusion-expr-common", + "datafusion-proto-models", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" +checksum = "f1a49dd3a76e4879012682c59dc3735282e60640c97483546a4113564a60bf8f" dependencies = [ "arrow", "datafusion-common", @@ -2165,15 +2214,16 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", - "itertools 0.14.0", + "datafusion-session", + "itertools 0.15.0", "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" +checksum = "b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97" dependencies = [ "arrow", "arrow-data", @@ -2181,6 +2231,7 @@ dependencies = [ "arrow-ord", "arrow-schema", "async-trait", + "bytes", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", @@ -2190,26 +2241,29 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", "futures", "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "num-traits", "parking_lot", "pin-project-lite", + "serde", + "serde_json", "tokio", ] [[package]] name = "datafusion-proto" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae" +checksum = "481881a3422072554c8c26a177a4f0a67f4d8c0e2a394a79e551e6bd9a5cec4b" dependencies = [ "arrow", - "chrono", "datafusion-catalog", "datafusion-catalog-listing", "datafusion-common", @@ -2225,26 +2279,38 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-proto-common", + "datafusion-proto-models", "object_store", "prost 0.14.4", ] [[package]] name = "datafusion-proto-common" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530" +checksum = "a95763d5fa64a88b658ba4ec7314d9c538e2aea749dc88d5ce2afaafbb4c69b8" dependencies = [ "arrow", "datafusion-common", "prost 0.14.4", ] +[[package]] +name = "datafusion-proto-models" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c4982d05dee1e0b90b7f3b39845dca7eee5635c550db09158d385b8181bdc6a" +dependencies = [ + "datafusion-common", + "datafusion-proto-common", + "prost 0.14.4", +] + [[package]] name = "datafusion-pruning" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" +checksum = "f11f0dbe2b5a642f6106e44de8bbd6654fa7220ca14f32313d4efd5514a86968" dependencies = [ "arrow", "datafusion-common", @@ -2258,10 +2324,11 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" +checksum = "62dc5ce37a5ce1ff5362949776829aded2529a11dbaa13d24b0ffa6a059e106a" dependencies = [ + "arrow-schema", "async-trait", "datafusion-common", "datafusion-execution", @@ -2272,23 +2339,22 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957" +checksum = "0c3d6f76463984c4ed8587c52ed4f21f142fd96fa55d8f128f673f2db4c46942" dependencies = [ "arrow", "bigdecimal", "chrono", "crc32fast", "datafusion", - "datafusion-catalog", "datafusion-common", - "datafusion-execution", "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-aggregate-common", "datafusion-functions-nested", + "datafusion-session", "log", "num-traits", "percent-encoding", @@ -2302,9 +2368,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" +checksum = "816c224a0ef5469a32a562b5d4830544f714f97c5f50142109286d40a288c7db" dependencies = [ "arrow", "bigdecimal", @@ -2317,6 +2383,7 @@ dependencies = [ "recursive", "regex", "sqlparser", + "stacker", ] [[package]] @@ -2597,14 +2664,26 @@ checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" [[package]] name = "fastlanes" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9272a674b446f53253f66582596614e41552d14968a3d1aaa590576a91c1188" +checksum = "4a516528425408429de7661e7101f7e7ad54f7d3fab859a3a6de239f508d17e2" dependencies = [ "const_for", - "core_detect", "num-traits", - "paste", + "pastey", + "seq-macro", +] + +[[package]] +name = "fastlanes" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcb34a73abd2872401096bf65d13e8d26e0c9ca6166d5e2b11a314ba017cdef" +dependencies = [ + "const_for", + "cpufeatures 0.3.0", + "num-traits", + "pastey", "seq-macro", ] @@ -2614,6 +2693,23 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fearless_simd" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629c9d6b6f30b5604089e0980f4d924a013aacb1654a6cdf5d3d852224f68651" + +[[package]] +name = "fearless_simd_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e91d94c6885390b81782d7ee60fd6583b33907333a8b340de25ccf468353315" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "filetime" version = "0.2.29" @@ -2717,9 +2813,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst-rs" -version = "0.5.11" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b13ac798afc0d9194eb4efefef8b9332efbd80b43f302a968cb8cb23b9d5360" +checksum = "03b614a2a7f2efc50d76c1e47d64b57c55acf6e0e718b0cae78236d147c487d0" dependencies = [ "rustc-hash", ] @@ -3443,12 +3539,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "inventory" version = "0.3.24" @@ -3488,6 +3578,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -3916,9 +4015,6 @@ name = "lz4_flex" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" -dependencies = [ - "twox-hash", -] [[package]] name = "lz4_flex" @@ -4179,7 +4275,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ - "num-bigint", + "num-bigint 0.4.8", "num-complex", "num-integer", "num-iter", @@ -4198,6 +4294,16 @@ dependencies = [ "serde", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint-dig" version = "0.8.6" @@ -4254,7 +4360,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "num-bigint", + "num-bigint 0.4.8", "num-integer", "num-traits", ] @@ -4365,10 +4471,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" [[package]] -name = "oneshot" +name = "onpair" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe21416a02c693fb9f980befcb230ecc70b0b3d1cc4abf88b9675c4c1457f0c" +checksum = "34272d1ea63266148a9a5b2b96db80a45245d034862302735f2f93413a6f9172" +dependencies = [ + "hashbrown 0.16.1", + "rand 0.9.5", +] [[package]] name = "opendal-core" @@ -4619,9 +4729,9 @@ dependencies = [ [[package]] name = "orc-rust" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b9867c4e7343218682ba11aceae1310c80735ec2beaf6124a0f8f848dad197" +checksum = "e54c163f843c4f0fcfca65b9b73ca6f93c37c8e86736d5a1e578c6e843eed0a0" dependencies = [ "arrow", "async-trait", @@ -4644,15 +4754,6 @@ dependencies = [ "zstd 0.13.3", ] -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - [[package]] name = "ordered-float" version = "5.3.0" @@ -4725,7 +4826,7 @@ dependencies = [ "lz4_flex 0.14.0", "lzokay-native", "md-5 0.10.6", - "num-bigint", + "num-bigint 0.4.8", "num-traits", "opendal-core", "opendal-http-transport-reqwest", @@ -4859,13 +4960,14 @@ dependencies = [ [[package]] name = "paimon-mosaic-core" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d2799ba43edf953feaf552dc48aee63c5923f774b7277473f4884e055442de" +checksum = "6127799590f9e9d6c552d73690b79e60b43bba5a72a949c9cd44e2064ec6d245" dependencies = [ "arrow-array", "arrow-buffer", "arrow-schema", + "libc", "zstd 0.13.3", ] @@ -4947,9 +5049,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" +checksum = "8eb260d566f2fbdabe356d26aaa5d13403aac29b5b4e44463b8f9bd7b2b12680" dependencies = [ "ahash", "arrow-array", @@ -4958,7 +5060,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.22.1", + "base64 0.23.1", "brotli", "bytes", "chrono", @@ -4966,27 +5068,78 @@ dependencies = [ "futures", "half", "hashbrown 0.17.1", - "lz4_flex 0.13.1", - "num-bigint", + "lz4_flex 0.14.0", + "num-bigint 0.5.1", "num-integer", "num-traits", "object_store", - "paste", "seq-macro", "simdutf8", "snap", - "thrift", "tokio", "twox-hash", "zstd 0.13.3", ] +[[package]] +name = "parquet-variant" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d0085fde95ccfae0b8d703acd1245bda8855a0bf996eba848e8e2f9995f8606" +dependencies = [ + "arrow", + "arrow-schema", + "chrono", + "half", + "indexmap 2.14.0", + "num-traits", + "simdutf8", + "uuid", +] + +[[package]] +name = "parquet-variant-compute" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1f9036efdea8177c84895f409a442e3960a8ccc642792760d8d48415542bb3a" +dependencies = [ + "arrow", + "arrow-schema", + "chrono", + "half", + "indexmap 2.14.0", + "parquet-variant", + "parquet-variant-json", + "serde_json", + "uuid", +] + +[[package]] +name = "parquet-variant-json" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50ae7a6fdff86cdd5075653039b62e2ba674b0aae9228022cd74dc9cf71d7353" +dependencies = [ + "arrow-schema", + "base64 0.23.1", + "chrono", + "parquet-variant", + "serde_json", + "uuid", +] + [[package]] name = "paste" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "pbkdf2" version = "0.12.2" @@ -5370,9 +5523,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.28.3" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +checksum = "700d18fa267b73b9b521fd7e13580e2f446916f176cacee1ab63fcc8191f1655" dependencies = [ "chrono", "libc", @@ -5385,18 +5538,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.3" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +checksum = "7b3fc0c4d08f6bb10e71fe39dfb9e2f59c6eb6854e22ec8092f50c69a4499adb" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.28.3" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +checksum = "dfc0b8e19df29aad7086cf977bb0c2a2f143e30567eb113e9cf72b62ca698330" dependencies = [ "libc", "pyo3-build-config", @@ -5404,9 +5557,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.3" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +checksum = "6100e8a4b5eba53afaa5ed078364851a0b2499c553a44c31026b929049b49dc6" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -5416,13 +5569,12 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.28.3" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +checksum = "6143877a16e82b5a727b7127ff4cd86858a24a28d745d72f43e6f227c7b1bdb3" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", "quote", "syn 2.0.119", ] @@ -5724,7 +5876,7 @@ checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.9", ] [[package]] @@ -6341,7 +6493,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.9", ] [[package]] @@ -6376,7 +6528,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.9", ] [[package]] @@ -6799,9 +6951,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.2" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" dependencies = [ "proc-macro2", "quote", @@ -6884,7 +7036,7 @@ dependencies = [ "measure_time", "memmap2", "once_cell", - "oneshot 0.1.13", + "oneshot", "rayon", "regex", "rust-stemmers", @@ -6976,7 +7128,7 @@ checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82" dependencies = [ "fnv", "nom", - "ordered-float 5.3.0", + "ordered-float", "serde", "serde_json", ] @@ -7093,18 +7245,7 @@ checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", -] - -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "ordered-float 2.10.1", + "syn 3.0.9", ] [[package]] @@ -7425,7 +7566,7 @@ checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.9", ] [[package]] @@ -7508,79 +7649,72 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vortex" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b722c8baa7328544d3ce22763321a9be97a75f11acdb630bb71abbe61def2a" +checksum = "00e778cec317a66ad43111508c9dfcc782af9bbac8ace68097651d687f864204" dependencies = [ "vortex-alp", "vortex-array", + "vortex-arrow", "vortex-btrblocks", "vortex-buffer", "vortex-bytebool", "vortex-datetime-parts", "vortex-decimal-byte-parts", + "vortex-edition", "vortex-error", "vortex-fastlanes", "vortex-file", - "vortex-flatbuffers", "vortex-fsst", "vortex-io", "vortex-ipc", "vortex-layout", "vortex-mask", "vortex-metrics", + "vortex-parquet-variant", "vortex-pco", - "vortex-proto", "vortex-runend", "vortex-scan", "vortex-sequence", "vortex-session", "vortex-sparse", "vortex-utils", + "vortex-uuid", "vortex-zigzag", "vortex-zstd", ] [[package]] name = "vortex-alp" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f1c242a300d0bef65ebcbaaf0086ccff778ac45e62f0bdcefbfcea5b078c979" +checksum = "1c0f717b7cc81f1cd0e94f017c04d6c91ffc9240143d14830415aad948e1e90e" dependencies = [ + "alp", "itertools 0.14.0", "num-traits", "prost 0.14.4", - "rustc-hash", "vortex-array", "vortex-buffer", "vortex-error", "vortex-fastlanes", "vortex-mask", "vortex-session", - "vortex-utils", ] [[package]] name = "vortex-array" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d33d2bed6547a9a30ddd4ae35259ffaca66bd3deac734f340fa4f98c046eb297" +checksum = "454098724288d8e27e99e0c8bd8875570ea7ce6add928024e701e99bd936d9bb" dependencies = [ - "arc-swap", "arcref", - "arrow-arith", - "arrow-array", "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ord", - "arrow-schema", - "arrow-select", - "arrow-string", - "async-lock", "bytes", "cfg-if", "enum-iterator", + "fearless_simd", + "fearless_simd_macros", "flatbuffers", "futures", "half", @@ -7588,49 +7722,73 @@ dependencies = [ "inventory", "itertools 0.14.0", "jiff", + "memchr", "num-traits", "num_enum", "parking_lot", "paste", "pin-project-lite", "prost 0.14.4", + "prost-types", "rand 0.10.2", + "regex", + "regex-syntax", "rustc-hash", "simdutf8", "smallvec", "static_assertions", - "termtree", "tracing", - "uuid", "vortex-array-macros", "vortex-buffer", + "vortex-compute", "vortex-error", - "vortex-flatbuffers", "vortex-mask", - "vortex-proto", "vortex-session", "vortex-utils", ] [[package]] name = "vortex-array-macros" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57dba54aebd8dc42230bb4345ede660741c34fc10439fdb1369824cb86bae907" +checksum = "a2210cabfd84df126829b7607c09a16ebce7eb79fe9f346567f9321e24e787a8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] -name = "vortex-btrblocks" -version = "0.75.0" +name = "vortex-arrow" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "110427e1a65221519c8127e92a8080ee922b5f76a311568d227fd227bfbdfa10" +checksum = "1647f385ca0a718f4722fc6cdfa54dde716b2172a36d9f49ba812eb6be149b1d" dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", "itertools 0.14.0", "num-traits", + "simdutf8", + "tracing", + "vortex-array", + "vortex-buffer", + "vortex-error", + "vortex-mask", + "vortex-runend", + "vortex-session", +] + +[[package]] +name = "vortex-btrblocks" +version = "0.90.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1138d084c1b3988936312ba65262caafcfedeb1991c9937dabeb1051cab6dfb" +dependencies = [ + "itertools 0.14.0", "pco", "rand 0.10.2", "vortex-alp", @@ -7639,12 +7797,15 @@ dependencies = [ "vortex-compressor", "vortex-datetime-parts", "vortex-decimal-byte-parts", + "vortex-edition", "vortex-error", "vortex-fastlanes", "vortex-fsst", + "vortex-onpair", "vortex-pco", "vortex-runend", "vortex-sequence", + "vortex-session", "vortex-sparse", "vortex-utils", "vortex-zigzag", @@ -7653,37 +7814,40 @@ dependencies = [ [[package]] name = "vortex-buffer" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8da47e7e8afbaa18e4471a8f99f7c33e909d07f1e9cdfeae6df3c1e4f71070" +checksum = "db52df963f2caa4380abb5ed4405476b2fb5d1675ae60b35cb80791a2de209dc" dependencies = [ + "allocator-api2", "arrow-buffer", "bitvec", "bytes", + "fearless_simd", + "fearless_simd_macros", "itertools 0.14.0", + "num-traits", "simdutf8", "vortex-error", ] [[package]] name = "vortex-bytebool" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b26f84e77f32726ec826ac70582b37e34d89bf8f007dc0b3426a188d823f45a5" +checksum = "32269208b92c2aa76ee81fe629e0bc45698d8cca051712a97b5e6326b31a1f67" dependencies = [ "num-traits", "vortex-array", "vortex-buffer", "vortex-error", - "vortex-mask", "vortex-session", ] [[package]] name = "vortex-compressor" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daa039a03a9350e8ca3219bd141e852a139c0a89811c3a7caecc22fcc8d07d35" +checksum = "80d16c2cb63ce298a3a2858e43f7ae29105fd74b86bcecb52311f3a9fa8ae1b0" dependencies = [ "itertools 0.14.0", "num-traits", @@ -7698,16 +7862,26 @@ dependencies = [ "vortex-utils", ] +[[package]] +name = "vortex-compute" +version = "0.90.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1b6bb64c801e393f509a17c2afcea639f04baed3eca96af76d2b41bef91b870" +dependencies = [ + "vortex-buffer", +] + [[package]] name = "vortex-datetime-parts" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bade3b947e68b638ad184e24b1367c3146e394e4dc837486444eb0bab7cf578" +checksum = "a67e541dca82970bf5ea3ad5ea47b02ab46f60cd53a2ada94e561aeabef74d2d" dependencies = [ "num-traits", "prost 0.14.4", "vortex-array", "vortex-buffer", + "vortex-compute", "vortex-error", "vortex-mask", "vortex-session", @@ -7715,9 +7889,9 @@ dependencies = [ [[package]] name = "vortex-decimal-byte-parts" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196c22cb318d0cf2a67eb7cf9b93550eb394a0f4f56775c119abbc3fa5e9c4c2" +checksum = "08e3371e16e9ee8cc24734d9e9e67cf8013ec2a4a443fe499916edb11aba780b" dependencies = [ "num-traits", "prost 0.14.4", @@ -7728,11 +7902,22 @@ dependencies = [ "vortex-session", ] +[[package]] +name = "vortex-edition" +version = "0.90.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecf0598d26fffb33ed48cb4d5e2ad8cf7a11ac929965d20a6959047c65ba7d1" +dependencies = [ + "parking_lot", + "vortex-error", + "vortex-session", +] + [[package]] name = "vortex-error" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80158c6f4397a08dc27c14ec46c3ffbecf18d0690841e5ffe9aef2d3a0a241ab" +checksum = "ade3532eb43eb26ccbf0a648165b2e2c3e8f64843b0648744bac9f51aa08a18c" dependencies = [ "arrow-schema", "flatbuffers", @@ -7743,17 +7928,18 @@ dependencies = [ [[package]] name = "vortex-fastlanes" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312a4446fbd20cbbb395a16ec3428d8b113c9b1188c51fd4c0b6265ccc487149" +checksum = "85b39609df49674c7df3841fba541cd61775d15a07aaad2fb124821d0ab44892" dependencies = [ - "fastlanes", + "fastlanes 0.7.2", "itertools 0.14.0", "lending-iterator", "num-traits", "prost 0.14.4", "vortex-array", "vortex-buffer", + "vortex-compute", "vortex-error", "vortex-mask", "vortex-session", @@ -7761,9 +7947,9 @@ dependencies = [ [[package]] name = "vortex-file" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1b30147596131d56ebaeb0838372faa09621302e916da553e4413aad96ce0c" +checksum = "225844b04952557b9ca8622867fd2f0749a2577385148b695f71d9f9f0a9af1d" dependencies = [ "async-trait", "bytes", @@ -7772,11 +7958,11 @@ dependencies = [ "itertools 0.14.0", "kanal", "moka", - "oneshot 0.2.1", "parking_lot", "pin-project-lite", "tokio", "tracing", + "url", "vortex-alp", "vortex-array", "vortex-btrblocks", @@ -7784,14 +7970,15 @@ dependencies = [ "vortex-bytebool", "vortex-datetime-parts", "vortex-decimal-byte-parts", + "vortex-edition", "vortex-error", "vortex-fastlanes", - "vortex-flatbuffers", "vortex-fsst", "vortex-io", "vortex-layout", "vortex-mask", "vortex-metrics", + "vortex-onpair", "vortex-pco", "vortex-runend", "vortex-scan", @@ -7803,24 +7990,14 @@ dependencies = [ "vortex-zstd", ] -[[package]] -name = "vortex-flatbuffers" -version = "0.75.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ecc9db16991f005079050cd2245dac110eb2cc321e9c087a530acc09b423bb0" -dependencies = [ - "flatbuffers", - "vortex-buffer", - "vortex-error", -] - [[package]] name = "vortex-fsst" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "570382994bd8bc21836550f04427a8d85f419f9e897c9ec042c1486ab34aa063" +checksum = "e2a198ab5fcd335db05dab36506ae37988acde7e916d4bbf3cbac6519ae6ae27" dependencies = [ "fsst-rs", + "num-traits", "prost 0.14.4", "vortex-array", "vortex-buffer", @@ -7831,19 +8008,20 @@ dependencies = [ [[package]] name = "vortex-io" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1747e9615151214cd31ec00a854cd6dd4bf677d9f649b5b32f0d4d5f40c1e268" +checksum = "2157da7dbfa60b1303616b939e90f54faf4a02a777c245ed2d0a227edc42e5d4" dependencies = [ + "async-executor", "async-fs", "async-stream", "async-trait", "bytes", + "crossbeam-channel", "custom-labels", "futures", "glob", "kanal", - "oneshot 0.2.1", "parking_lot", "pin-project-lite", "smol", @@ -7860,9 +8038,9 @@ dependencies = [ [[package]] name = "vortex-ipc" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4bd60c8735ea6e0f71a27a16f44f9fbb8d293c341d30ad6f042103a2b4808b8" +checksum = "e8119836dbbfde9243895fc915eb64d17b4d52a63a48b5746fd17e112601c907" dependencies = [ "bytes", "flatbuffers", @@ -7872,19 +8050,35 @@ dependencies = [ "vortex-array", "vortex-buffer", "vortex-error", - "vortex-flatbuffers", + "vortex-session", +] + +[[package]] +name = "vortex-json" +version = "0.90.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab575eaab239140d1dce28783bffa8ec96642b5b7ca8370e311bd6f9a4930e3" +dependencies = [ + "arrow-array", + "arrow-schema", + "prost 0.14.4", + "vortex-array", + "vortex-arrow", + "vortex-edition", + "vortex-error", "vortex-session", ] [[package]] name = "vortex-layout" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "526570e7cfdf252798015e27cf175f2cac40aae1e164337fd382501193ec4274" +checksum = "2ae8a8e5da4d59590c5db1d580988c74619bbfe39cd18df47660f2fcbf6f4f7e" dependencies = [ "arcref", "arrow-array", "arrow-schema", + "async-lock", "async-stream", "async-trait", "bit-vec", @@ -7894,21 +8088,24 @@ dependencies = [ "kanal", "moka", "once_cell", - "oneshot 0.2.1", "parking_lot", "paste", "pin-project-lite", "prost 0.14.4", + "roaring", "rustc-hash", "sketches-ddsketch", + "smallvec", "termtree", "tokio", "tracing", + "twox-hash", + "uuid", "vortex-array", + "vortex-arrow", "vortex-btrblocks", "vortex-buffer", "vortex-error", - "vortex-flatbuffers", "vortex-io", "vortex-mask", "vortex-metrics", @@ -7921,9 +8118,9 @@ dependencies = [ [[package]] name = "vortex-mask" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5b43ee2a50c722d9abdaed454e836b044a3a69acbb5f5d95e28b89e520b11d9" +checksum = "d7b17f6601def0aad3bebebb175219f09b59c2e137860962af078e4e55e52e9c" dependencies = [ "itertools 0.14.0", "vortex-buffer", @@ -7932,21 +8129,22 @@ dependencies = [ [[package]] name = "vortex-metrics" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e079c9fe032af05b2e38bf91e63d2bdf36bd01e3920bc4a352648b2b21e9f70" +checksum = "8f0d03b9b48c72005f71f60c7134d1fcac3985e034b914a56f54e2a223097bb7" dependencies = [ "parking_lot", "sketches-ddsketch", ] [[package]] -name = "vortex-pco" -version = "0.75.0" +name = "vortex-onpair" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8902b2df3068b37267c927cdec8905ea4e0e693d81274ba9713000929d1e514e" +checksum = "d7b86706bb011bd76084e5fb0f34137a10fa210a8364b28cdfb53dd7222a2ab3" dependencies = [ - "pco", + "num-traits", + "onpair", "prost 0.14.4", "vortex-array", "vortex-buffer", @@ -7956,22 +8154,48 @@ dependencies = [ ] [[package]] -name = "vortex-proto" -version = "0.75.0" +name = "vortex-parquet-variant" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f018abac72332c320ba3b8b59b6ce72f6c8a6caf0552b0b5c255f336b7f502" +checksum = "253d55452deb61da604207bbd557de421a962c933197682cfe4ee72a3e9934e2" dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "chrono", + "parquet-variant", + "parquet-variant-compute", "prost 0.14.4", - "prost-types", + "vortex-array", + "vortex-arrow", + "vortex-buffer", + "vortex-error", + "vortex-json", + "vortex-mask", + "vortex-session", +] + +[[package]] +name = "vortex-pco" +version = "0.90.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d54b4a848b899a70fcbbd86da1e20b4ec0527722a08dbf3cf000610f7f02260" +dependencies = [ + "pco", + "prost 0.14.4", + "vortex-array", + "vortex-buffer", + "vortex-error", + "vortex-mask", + "vortex-session", ] [[package]] name = "vortex-runend" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73dd8c96206fd4c0110e9ac304fd41e78f6e85659975c9e63911dddd982de5fd" +checksum = "2828cdf7c3096227136ca26b93c1ec5b55c4b74f45a2bff0ea98ec334adf2bd1" dependencies = [ - "arrow-array", "itertools 0.14.0", "num-traits", "prost 0.14.4", @@ -7984,9 +8208,9 @@ dependencies = [ [[package]] name = "vortex-scan" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34deb979fc801bbc0e015ceccca1870f1dbbc72426fc040d277ea86cd9df5fd" +checksum = "c966f342f8251c796dee968ad1a88221f9bf1e94fda1744b5094ab6e05c6b3b3" dependencies = [ "async-trait", "futures", @@ -7995,34 +8219,36 @@ dependencies = [ "vortex-array", "vortex-buffer", "vortex-error", + "vortex-io", "vortex-mask", "vortex-session", + "vortex-utils", ] [[package]] name = "vortex-sequence" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "933091989818d88836430fb68ca0b3ef9e054bf2740f94051377231cc287d427" +checksum = "f0913adc59e97fb965b9bcf5feee11bf090027b7c7b711494c4b72282365e1bd" dependencies = [ "num-traits", "prost 0.14.4", "smallvec", "vortex-array", "vortex-buffer", + "vortex-compute", "vortex-error", "vortex-mask", - "vortex-proto", "vortex-session", ] [[package]] name = "vortex-session" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a62810f3a266cdd8bc4812235b8d2b1b586a909dc69107ba51f4dbfee01d9f2" +checksum = "03fcffcfac13c4d854f836dc584d1611cc8aba473e728f804c0607e58ee88238" dependencies = [ - "dashmap", + "arc-swap", "lasso", "parking_lot", "vortex-error", @@ -8031,9 +8257,9 @@ dependencies = [ [[package]] name = "vortex-sparse" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe0d321f522d325a811bfc6506b320db12f57bd7710d558520d75dd7313b9dd1" +checksum = "b9c445dddd2c49ce562444382be4d4cac8a36d2832a0a1b2728acbd4b74b107a" dependencies = [ "itertools 0.14.0", "num-traits", @@ -8047,23 +8273,39 @@ dependencies = [ [[package]] name = "vortex-utils" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebe05be4a3e14c562c80f78cbdec0e5a05030e4d1299a6ca6609441758414202" +checksum = "8fd7be99f740319f84ddd5176535b3616172c99fb7b3e3f2301ec7eb8de053d1" dependencies = [ "dashmap", "hashbrown 0.17.1", +] + +[[package]] +name = "vortex-uuid" +version = "0.90.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3252fb4740197481d86249581a6fe536db6879af5d17a64fdd82a624d16245e" +dependencies = [ + "arrow-array", + "arrow-schema", + "uuid", + "vortex-array", + "vortex-arrow", + "vortex-buffer", "vortex-error", + "vortex-session", ] [[package]] name = "vortex-zigzag" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6635049187e7e46413fc2d760a2836b73c0b1ffcbbc54b657b6c338a5551c5c6" +checksum = "83e07baa4d5952093dbd271cda75ae01e787ecdca1e59394519b30411ed28303" dependencies = [ "vortex-array", "vortex-buffer", + "vortex-compute", "vortex-error", "vortex-mask", "vortex-session", @@ -8072,14 +8314,16 @@ dependencies = [ [[package]] name = "vortex-zstd" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dc464ba75cff8510a7f31e8c5aa7ca4f3ce34d15e87e80a91880d3afa7458df" +checksum = "16ed140fdf797d28c36b4d4b73223a7874d794ecd3c9cd3a79e16977bd6b03b4" dependencies = [ "itertools 0.14.0", + "num-traits", "prost 0.14.4", "vortex-array", "vortex-buffer", + "vortex-edition", "vortex-error", "vortex-mask", "vortex-session", diff --git a/Cargo.toml b/Cargo.toml index 665c914a5..6574b5754 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,21 +36,21 @@ license = "Apache-2.0" rust-version = "1.95.0" [workspace.dependencies] -arrow = "58.0" -arrow-array = { version = "58.0", features = ["ffi"] } -arrow-arith = "58.0" -arrow-buffer = "58.0" -arrow-schema = "58.0" -arrow-cast = "58.0" -arrow-ord = "58.0" -arrow-row = "58.0" -arrow-select = "58.0" -arrow-string = "58.0" -datafusion = "54.0.0" -datafusion-ffi = "54.0.0" -datafusion-spark = { version = "54.0.0", features = ["core"] } +arrow = "59.2" +arrow-array = { version = "59.2", features = ["ffi"] } +arrow-arith = "59.2" +arrow-buffer = "59.2" +arrow-schema = "59.2" +arrow-cast = "59.2" +arrow-ord = "59.2" +arrow-row = "59.2" +arrow-select = "59.2" +arrow-string = "59.2" +datafusion = "55.2.0" +datafusion-ffi = "55.2.0" +datafusion-spark = { version = "55.2.0", features = ["core"] } paimon = { version = "0.6.0", path = "crates/paimon" } -parquet = "58.0" +parquet = "59.2" constant_time_eq = ">=0.4.0, <0.5.0" tokio = "1.39.2" tokio-util = "0.7" diff --git a/DEPENDENCIES.rust.tsv b/DEPENDENCIES.rust.tsv index 810dccaed..0e39b53ff 100644 --- a/DEPENDENCIES.rust.tsv +++ b/DEPENDENCIES.rust.tsv @@ -8,6 +8,7 @@ aho-corasick@1.1.4 X X alloc-no-stdlib@2.0.4 X alloc-stdlib@0.2.4 X allocator-api2@0.2.21 X X +alp@0.0.4 X android_system_properties@0.1.5 X X anstream@1.0.0 X X anstyle@1.0.14 X X @@ -22,21 +23,21 @@ arc-swap@1.9.2 X X arcref@0.2.0 X arrayref@0.3.9 X arrayvec@0.7.8 X X -arrow@58.4.0 X -arrow-arith@58.4.0 X -arrow-array@58.4.0 X X -arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X -arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-pyarrow@58.4.0 X -arrow-row@58.4.0 X -arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X +arrow-array@59.3.2 X X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-pyarrow@59.3.2 X +arrow-row@59.3.2 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-channel@2.5.0 X X async-compression@0.4.42 X X async-executor@1.14.0 X X @@ -67,7 +68,7 @@ base64ct@1.8.3 X X better_io@0.2.0 X bigdecimal@0.4.10 X X bindgen@0.72.1 X -bit-vec@0.9.1 X X +bit-vec@0.10.1 X X bitflags@2.13.1 X X bitpacking@0.9.3 X bitvec@1.1.1 X @@ -122,7 +123,6 @@ constant_time_eq@0.4.2 X X X core-foundation@0.10.1 X X core-foundation@0.9.4 X X core-foundation-sys@0.8.7 X X -core_detect@1.0.0 X X cpubits@0.1.1 X X cpufeatures@0.2.17 X X cpufeatures@0.3.0 X X @@ -147,41 +147,42 @@ darling_core@0.23.0 X darling_macro@0.23.0 X dary_heap@0.3.9 X X dashmap@6.2.1 X -datafusion@54.0.0 X -datafusion-catalog@54.0.0 X -datafusion-catalog-listing@54.0.0 X -datafusion-common@54.0.0 X -datafusion-common-runtime@54.0.0 X -datafusion-datasource@54.0.0 X -datafusion-datasource-arrow@54.0.0 X -datafusion-datasource-csv@54.0.0 X -datafusion-datasource-json@54.0.0 X -datafusion-datasource-parquet@54.0.0 X -datafusion-doc@54.0.0 X -datafusion-execution@54.0.0 X -datafusion-expr@54.0.0 X -datafusion-expr-common@54.0.0 X -datafusion-ffi@54.0.0 X -datafusion-functions@54.0.0 X -datafusion-functions-aggregate@54.0.0 X -datafusion-functions-aggregate-common@54.0.0 X -datafusion-functions-nested@54.0.0 X -datafusion-functions-table@54.0.0 X -datafusion-functions-window@54.0.0 X -datafusion-functions-window-common@54.0.0 X -datafusion-macros@54.0.0 X -datafusion-optimizer@54.0.0 X -datafusion-physical-expr@54.0.0 X -datafusion-physical-expr-adapter@54.0.0 X -datafusion-physical-expr-common@54.0.0 X -datafusion-physical-optimizer@54.0.0 X -datafusion-physical-plan@54.0.0 X -datafusion-proto@54.0.0 X -datafusion-proto-common@54.0.0 X -datafusion-pruning@54.0.0 X -datafusion-session@54.0.0 X -datafusion-spark@54.0.0 X -datafusion-sql@54.0.0 X +datafusion@55.2.0 X +datafusion-catalog@55.2.0 X +datafusion-catalog-listing@55.2.0 X +datafusion-common@55.2.0 X +datafusion-common-runtime@55.2.0 X +datafusion-datasource@55.2.0 X +datafusion-datasource-arrow@55.2.0 X +datafusion-datasource-csv@55.2.0 X +datafusion-datasource-json@55.2.0 X +datafusion-datasource-parquet@55.2.0 X +datafusion-doc@55.2.0 X +datafusion-execution@55.2.0 X +datafusion-expr@55.2.0 X +datafusion-expr-common@55.2.0 X +datafusion-ffi@55.2.0 X +datafusion-functions@55.2.0 X +datafusion-functions-aggregate@55.2.0 X +datafusion-functions-aggregate-common@55.2.0 X +datafusion-functions-nested@55.2.0 X +datafusion-functions-table@55.2.0 X +datafusion-functions-window@55.2.0 X +datafusion-functions-window-common@55.2.0 X +datafusion-macros@55.2.0 X +datafusion-optimizer@55.2.0 X +datafusion-physical-expr@55.2.0 X +datafusion-physical-expr-adapter@55.2.0 X +datafusion-physical-expr-common@55.2.0 X +datafusion-physical-optimizer@55.2.0 X +datafusion-physical-plan@55.2.0 X +datafusion-proto@55.2.0 X +datafusion-proto-common@55.2.0 X +datafusion-proto-models@55.2.0 X +datafusion-pruning@55.2.0 X +datafusion-session@55.2.0 X +datafusion-spark@55.2.0 X +datafusion-sql@55.2.0 X datasketches@0.2.0 X der@0.7.10 X X deranged@0.5.8 X X @@ -209,8 +210,11 @@ ext-trait-proc_macros@1.0.1 X X X extension-traits@1.0.1 X X X fallible-streaming-iterator@0.1.9 X X fastdivide@0.4.2 X X -fastlanes@0.5.2 X +fastlanes@0.6.1 X +fastlanes@0.7.2 X fastrand@2.5.0 X X +fearless_simd@1.2.0 X X +fearless_simd_macros@0.1.0 X X find-msvc-tools@0.1.9 X X fixedbitset@0.5.7 X X flatbuffers@25.12.19 X @@ -223,7 +227,7 @@ foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs4@0.13.1 X X fs_extra@1.3.0 X -fsst-rs@0.5.11 X +fsst-rs@0.6.0 X funty@2.0.0 X futures@0.3.33 X X futures-channel@0.3.33 X X @@ -287,12 +291,12 @@ include-flate-compress@0.3.4 X indexmap@2.14.0 X X inout@0.1.4 X X inout@0.2.2 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X is_terminal_polyfill@1.70.2 X X itertools@0.13.0 X X itertools@0.14.0 X X +itertools@0.15.0 X X itoa@1.0.18 X X jieba-macros@0.10.3 X jieba-rs@0.10.3 X @@ -364,6 +368,7 @@ nougat@0.2.4 X X X nougat-proc_macros@0.2.4 X X X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-bigint-dig@0.8.6 X X num-complex@0.4.6 X X num-conv@0.2.2 X X @@ -380,7 +385,7 @@ object_store@0.13.2 X X once_cell@1.21.4 X X once_cell_polyfill@1.70.2 X X oneshot@0.1.13 X X -oneshot@0.2.1 X X +onpair@0.2.1 X opendal-core@0.59.5 X opendal-http-transport-reqwest@0.59.5 X opendal-layer-retry@0.59.5 X @@ -397,8 +402,7 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X @@ -407,15 +411,19 @@ paimon-c@0.6.0 X paimon-datafusion@0.6.0 X paimon-ftindex-core@0.1.0 X paimon-integration-tests@0.6.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-rest-server@0.6.0 X paimon-tpcds-bench@0.6.0 X paimon-vindex-core@0.5.0 X parking@2.2.1 X X parking_lot@0.12.5 X X parking_lot_core@0.9.12 X X -parquet@58.4.0 X +parquet@59.3.2 X +parquet-variant@59.3.2 X +parquet-variant-compute@59.3.2 X +parquet-variant-json@59.3.2 X paste@1.0.15 X X +pastey@0.2.3 X X pbkdf2@0.12.2 X X pco@1.0.2 X pem@4.0.0 X @@ -455,11 +463,11 @@ prost-derive@0.13.5 X prost-derive@0.14.4 X prost-types@0.14.4 X psm@0.1.31 X X -pyo3@0.28.3 X X -pyo3-build-config@0.28.3 X X -pyo3-ffi@0.28.3 X X -pyo3-macros@0.28.3 X X -pyo3-macros-backend@0.28.3 X X +pyo3@0.29.3 X X +pyo3-build-config@0.29.3 X X +pyo3-ffi@0.29.3 X X +pyo3-macros@0.29.3 X X +pyo3-macros-backend@0.29.3 X X pypaimon_rust@0.6.0 X quad-rand@0.2.3 X quick-xml@0.41.0 X @@ -576,7 +584,7 @@ strum_macros@0.27.2 X subtle@2.6.1 X syn@1.0.109 X X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X @@ -600,7 +608,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X @@ -640,36 +647,41 @@ utf8parse@0.2.2 X X uuid@1.24.0 X X vcpkg@0.2.15 X X version_check@0.9.5 X X -vortex@0.75.0 X -vortex-alp@0.75.0 X -vortex-array@0.75.0 X -vortex-array-macros@0.75.0 X -vortex-btrblocks@0.75.0 X -vortex-buffer@0.75.0 X -vortex-bytebool@0.75.0 X -vortex-compressor@0.75.0 X -vortex-datetime-parts@0.75.0 X -vortex-decimal-byte-parts@0.75.0 X -vortex-error@0.75.0 X -vortex-fastlanes@0.75.0 X -vortex-file@0.75.0 X -vortex-flatbuffers@0.75.0 X -vortex-fsst@0.75.0 X -vortex-io@0.75.0 X -vortex-ipc@0.75.0 X -vortex-layout@0.75.0 X -vortex-mask@0.75.0 X -vortex-metrics@0.75.0 X -vortex-pco@0.75.0 X -vortex-proto@0.75.0 X -vortex-runend@0.75.0 X -vortex-scan@0.75.0 X -vortex-sequence@0.75.0 X -vortex-session@0.75.0 X -vortex-sparse@0.75.0 X -vortex-utils@0.75.0 X -vortex-zigzag@0.75.0 X -vortex-zstd@0.75.0 X +vortex@0.90.0 X +vortex-alp@0.90.0 X +vortex-array@0.90.0 X +vortex-array-macros@0.90.0 X +vortex-arrow@0.90.0 X +vortex-btrblocks@0.90.0 X +vortex-buffer@0.90.0 X +vortex-bytebool@0.90.0 X +vortex-compressor@0.90.0 X +vortex-compute@0.90.0 X +vortex-datetime-parts@0.90.0 X +vortex-decimal-byte-parts@0.90.0 X +vortex-edition@0.90.0 X +vortex-error@0.90.0 X +vortex-fastlanes@0.90.0 X +vortex-file@0.90.0 X +vortex-fsst@0.90.0 X +vortex-io@0.90.0 X +vortex-ipc@0.90.0 X +vortex-json@0.90.0 X +vortex-layout@0.90.0 X +vortex-mask@0.90.0 X +vortex-metrics@0.90.0 X +vortex-onpair@0.90.0 X +vortex-parquet-variant@0.90.0 X +vortex-pco@0.90.0 X +vortex-runend@0.90.0 X +vortex-scan@0.90.0 X +vortex-sequence@0.90.0 X +vortex-session@0.90.0 X +vortex-sparse@0.90.0 X +vortex-utils@0.90.0 X +vortex-uuid@0.90.0 X +vortex-zigzag@0.90.0 X +vortex-zstd@0.90.0 X walkdir@2.5.0 X X want@0.3.1 X wasi@0.11.1+wasi-snapshot-preview1 X X X diff --git a/benchmarks/tpcds/DEPENDENCIES.rust.tsv b/benchmarks/tpcds/DEPENDENCIES.rust.tsv index e1821d263..f63518a34 100644 --- a/benchmarks/tpcds/DEPENDENCIES.rust.tsv +++ b/benchmarks/tpcds/DEPENDENCIES.rust.tsv @@ -19,20 +19,20 @@ ar_archive_writer@0.5.2 X arc-swap@1.9.2 X X arrayref@0.3.9 X arrayvec@0.7.8 X X -arrow@58.4.0 X -arrow-arith@58.4.0 X -arrow-array@58.4.0 X X -arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X -arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-row@58.4.0 X -arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X +arrow-array@59.3.2 X X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-row@59.3.2 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-compression@0.4.42 X X async-stream@0.3.6 X async-stream-impl@0.3.6 X @@ -106,38 +106,38 @@ darling_core@0.23.0 X darling_macro@0.23.0 X dary_heap@0.3.9 X X dashmap@6.2.1 X -datafusion@54.0.0 X -datafusion-catalog@54.0.0 X -datafusion-catalog-listing@54.0.0 X -datafusion-common@54.0.0 X -datafusion-common-runtime@54.0.0 X -datafusion-datasource@54.0.0 X -datafusion-datasource-arrow@54.0.0 X -datafusion-datasource-csv@54.0.0 X -datafusion-datasource-json@54.0.0 X -datafusion-datasource-parquet@54.0.0 X -datafusion-doc@54.0.0 X -datafusion-execution@54.0.0 X -datafusion-expr@54.0.0 X -datafusion-expr-common@54.0.0 X -datafusion-functions@54.0.0 X -datafusion-functions-aggregate@54.0.0 X -datafusion-functions-aggregate-common@54.0.0 X -datafusion-functions-nested@54.0.0 X -datafusion-functions-table@54.0.0 X -datafusion-functions-window@54.0.0 X -datafusion-functions-window-common@54.0.0 X -datafusion-macros@54.0.0 X -datafusion-optimizer@54.0.0 X -datafusion-physical-expr@54.0.0 X -datafusion-physical-expr-adapter@54.0.0 X -datafusion-physical-expr-common@54.0.0 X -datafusion-physical-optimizer@54.0.0 X -datafusion-physical-plan@54.0.0 X -datafusion-pruning@54.0.0 X -datafusion-session@54.0.0 X -datafusion-spark@54.0.0 X -datafusion-sql@54.0.0 X +datafusion@55.2.0 X +datafusion-catalog@55.2.0 X +datafusion-catalog-listing@55.2.0 X +datafusion-common@55.2.0 X +datafusion-common-runtime@55.2.0 X +datafusion-datasource@55.2.0 X +datafusion-datasource-arrow@55.2.0 X +datafusion-datasource-csv@55.2.0 X +datafusion-datasource-json@55.2.0 X +datafusion-datasource-parquet@55.2.0 X +datafusion-doc@55.2.0 X +datafusion-execution@55.2.0 X +datafusion-expr@55.2.0 X +datafusion-expr-common@55.2.0 X +datafusion-functions@55.2.0 X +datafusion-functions-aggregate@55.2.0 X +datafusion-functions-aggregate-common@55.2.0 X +datafusion-functions-nested@55.2.0 X +datafusion-functions-table@55.2.0 X +datafusion-functions-window@55.2.0 X +datafusion-functions-window-common@55.2.0 X +datafusion-macros@55.2.0 X +datafusion-optimizer@55.2.0 X +datafusion-physical-expr@55.2.0 X +datafusion-physical-expr-adapter@55.2.0 X +datafusion-physical-expr-common@55.2.0 X +datafusion-physical-optimizer@55.2.0 X +datafusion-physical-plan@55.2.0 X +datafusion-pruning@55.2.0 X +datafusion-session@55.2.0 X +datafusion-spark@55.2.0 X +datafusion-sql@55.2.0 X datasketches@0.2.0 X deranged@0.5.8 X X diff@0.1.13 X X @@ -220,11 +220,11 @@ include-flate@0.3.4 X include-flate-codegen@0.3.4 X include-flate-compress@0.3.4 X indexmap@2.14.0 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X is_terminal_polyfill@1.70.2 X X itertools@0.14.0 X X +itertools@0.15.0 X X itoa@1.0.18 X X jieba-macros@0.10.3 X jieba-rs@0.10.3 X @@ -283,6 +283,7 @@ no_std_io2@0.9.4 X X nom@7.1.3 X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-complex@0.4.6 X X num-conv@0.2.2 X X num-integer@0.1.46 X X @@ -303,20 +304,19 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X paimon@0.6.0 X paimon-datafusion@0.6.0 X paimon-ftindex-core@0.1.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-tpcds-bench@0.6.0 X paimon-vindex-core@0.5.0 X parking_lot@0.12.5 X X parking_lot_core@0.9.12 X X -parquet@58.4.0 X +parquet@59.3.2 X paste@1.0.15 X X percent-encoding@2.3.2 X X petgraph@0.8.3 X X @@ -431,7 +431,7 @@ strum@0.27.2 X strum_macros@0.27.2 X subtle@2.6.1 X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X @@ -451,7 +451,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X diff --git a/benchmarks/tpcds/src/lib.rs b/benchmarks/tpcds/src/lib.rs index b90f655dc..57ee72948 100644 --- a/benchmarks/tpcds/src/lib.rs +++ b/benchmarks/tpcds/src/lib.rs @@ -200,7 +200,12 @@ mod tests { 3 ); assert_eq!( - ctx.ctx().state().config_options().execution.batch_size, + ctx.ctx() + .state() + .config_options() + .execution + .batch_size + .get(), 4096 ); assert!( @@ -253,7 +258,7 @@ mod tests { assert_eq!(decoded.source, SourceKind::Paimon); assert!(decoded.disclosure.contains("non-TPC")); - assert_eq!(decoded.datafusion_version, "54.0.0"); + assert_eq!(decoded.datafusion_version, datafusion::DATAFUSION_VERSION); } #[test] diff --git a/bindings/c/DEPENDENCIES.rust.tsv b/bindings/c/DEPENDENCIES.rust.tsv index 8316c08d4..872dd10d7 100644 --- a/bindings/c/DEPENDENCIES.rust.tsv +++ b/bindings/c/DEPENDENCIES.rust.tsv @@ -12,20 +12,20 @@ anyhow@1.0.104 X X apache-avro@0.21.0 X approx@0.5.1 X arc-swap@1.9.2 X X -arrow@58.4.0 X -arrow-arith@58.4.0 X -arrow-array@58.4.0 X X -arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X -arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-row@58.4.0 X -arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X +arrow-array@59.3.2 X X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-row@59.3.2 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-compression@0.4.42 X X async-stream@0.3.6 X async-stream-impl@0.3.6 X @@ -174,7 +174,6 @@ include-flate-codegen@0.3.4 X include-flate-compress@0.3.4 X indexmap@2.14.0 X X inout@0.1.4 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X itertools@0.14.0 X X @@ -235,6 +234,7 @@ no_std_io2@0.9.4 X X nom@7.1.3 X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-bigint-dig@0.8.6 X X num-complex@0.4.6 X X num-conv@0.2.2 X X @@ -259,17 +259,16 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X paimon@0.6.0 X paimon-c@0.6.0 X paimon-ftindex-core@0.1.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-vindex-core@0.5.0 X -parquet@58.4.0 X +parquet@59.3.2 X paste@1.0.15 X X pbkdf2@0.12.2 X X pem@4.0.0 X @@ -391,7 +390,7 @@ strum@0.27.2 X strum_macros@0.27.2 X subtle@2.6.1 X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X @@ -411,7 +410,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X diff --git a/bindings/c/src/tests.rs b/bindings/c/src/tests.rs index 108078d4e..91a0f8e69 100644 --- a/bindings/c/src/tests.rs +++ b/bindings/c/src/tests.rs @@ -3324,6 +3324,13 @@ fn build_pk_vector_table(path: &str, vectors: &[[f32; PK_DIM]]) -> Table { ); crate::runtime().block_on(async { + // Automatic compaction reloads the schema from storage after committing. + file_io + .new_output(&table.schema_manager().schema_path(table.schema().id())) + .unwrap() + .write(serde_json::to_vec(table.schema()).unwrap().into()) + .await + .unwrap(); // Write a real data file via the public write path to obtain a genuine // DataFileMeta (name, row count, stats, file size). let write_builder = table.new_write_builder(); diff --git a/bindings/go/DEPENDENCIES.rust.tsv b/bindings/go/DEPENDENCIES.rust.tsv index 8316c08d4..872dd10d7 100644 --- a/bindings/go/DEPENDENCIES.rust.tsv +++ b/bindings/go/DEPENDENCIES.rust.tsv @@ -12,20 +12,20 @@ anyhow@1.0.104 X X apache-avro@0.21.0 X approx@0.5.1 X arc-swap@1.9.2 X X -arrow@58.4.0 X -arrow-arith@58.4.0 X -arrow-array@58.4.0 X X -arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X -arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-row@58.4.0 X -arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X +arrow-array@59.3.2 X X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-row@59.3.2 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-compression@0.4.42 X X async-stream@0.3.6 X async-stream-impl@0.3.6 X @@ -174,7 +174,6 @@ include-flate-codegen@0.3.4 X include-flate-compress@0.3.4 X indexmap@2.14.0 X X inout@0.1.4 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X itertools@0.14.0 X X @@ -235,6 +234,7 @@ no_std_io2@0.9.4 X X nom@7.1.3 X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-bigint-dig@0.8.6 X X num-complex@0.4.6 X X num-conv@0.2.2 X X @@ -259,17 +259,16 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X paimon@0.6.0 X paimon-c@0.6.0 X paimon-ftindex-core@0.1.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-vindex-core@0.5.0 X -parquet@58.4.0 X +parquet@59.3.2 X paste@1.0.15 X X pbkdf2@0.12.2 X X pem@4.0.0 X @@ -391,7 +390,7 @@ strum@0.27.2 X strum_macros@0.27.2 X subtle@2.6.1 X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X @@ -411,7 +410,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index bc459d552..2ae397cee 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -39,7 +39,7 @@ futures = "0.3" paimon = { path = "../../crates/paimon", features = ["storage-all", "internal-api"] } paimon-datafusion = { path = "../../crates/integrations/datafusion" } paimon-vindex-core = "0.5.0" -pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] } +pyo3 = { version = "0.29", features = ["abi3-py310", "chrono"] } roaring = "0.11" serde_json = "1.0" tokio = { workspace = true } @@ -47,4 +47,4 @@ tokio = { workspace = true } [dev-dependencies] # Enables `Python::attach` to auto-initialize an interpreter for Rust unit tests. # Only applied to the test build; the maturin/extension-module build does not pull dev-dependencies. -pyo3 = { version = "0.28", features = ["abi3-py310", "auto-initialize", "chrono"] } +pyo3 = { version = "0.29", features = ["abi3-py310", "auto-initialize", "chrono"] } diff --git a/bindings/python/DEPENDENCIES.rust.tsv b/bindings/python/DEPENDENCIES.rust.tsv index 642f4d3a9..a18d0382c 100644 --- a/bindings/python/DEPENDENCIES.rust.tsv +++ b/bindings/python/DEPENDENCIES.rust.tsv @@ -16,21 +16,21 @@ ar_archive_writer@0.5.2 X arc-swap@1.9.2 X X arrayref@0.3.9 X arrayvec@0.7.8 X X -arrow@58.4.0 X -arrow-arith@58.4.0 X -arrow-array@58.4.0 X X -arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X -arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-pyarrow@58.4.0 X -arrow-row@58.4.0 X -arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X +arrow-array@59.3.2 X X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-pyarrow@59.3.2 X +arrow-row@59.3.2 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-compression@0.4.42 X X async-ffi@0.5.0 X async-stream@0.3.6 X @@ -113,41 +113,42 @@ darling_core@0.23.0 X darling_macro@0.23.0 X dary_heap@0.3.9 X X dashmap@6.2.1 X -datafusion@54.0.0 X -datafusion-catalog@54.0.0 X -datafusion-catalog-listing@54.0.0 X -datafusion-common@54.0.0 X -datafusion-common-runtime@54.0.0 X -datafusion-datasource@54.0.0 X -datafusion-datasource-arrow@54.0.0 X -datafusion-datasource-csv@54.0.0 X -datafusion-datasource-json@54.0.0 X -datafusion-datasource-parquet@54.0.0 X -datafusion-doc@54.0.0 X -datafusion-execution@54.0.0 X -datafusion-expr@54.0.0 X -datafusion-expr-common@54.0.0 X -datafusion-ffi@54.0.0 X -datafusion-functions@54.0.0 X -datafusion-functions-aggregate@54.0.0 X -datafusion-functions-aggregate-common@54.0.0 X -datafusion-functions-nested@54.0.0 X -datafusion-functions-table@54.0.0 X -datafusion-functions-window@54.0.0 X -datafusion-functions-window-common@54.0.0 X -datafusion-macros@54.0.0 X -datafusion-optimizer@54.0.0 X -datafusion-physical-expr@54.0.0 X -datafusion-physical-expr-adapter@54.0.0 X -datafusion-physical-expr-common@54.0.0 X -datafusion-physical-optimizer@54.0.0 X -datafusion-physical-plan@54.0.0 X -datafusion-proto@54.0.0 X -datafusion-proto-common@54.0.0 X -datafusion-pruning@54.0.0 X -datafusion-session@54.0.0 X -datafusion-spark@54.0.0 X -datafusion-sql@54.0.0 X +datafusion@55.2.0 X +datafusion-catalog@55.2.0 X +datafusion-catalog-listing@55.2.0 X +datafusion-common@55.2.0 X +datafusion-common-runtime@55.2.0 X +datafusion-datasource@55.2.0 X +datafusion-datasource-arrow@55.2.0 X +datafusion-datasource-csv@55.2.0 X +datafusion-datasource-json@55.2.0 X +datafusion-datasource-parquet@55.2.0 X +datafusion-doc@55.2.0 X +datafusion-execution@55.2.0 X +datafusion-expr@55.2.0 X +datafusion-expr-common@55.2.0 X +datafusion-ffi@55.2.0 X +datafusion-functions@55.2.0 X +datafusion-functions-aggregate@55.2.0 X +datafusion-functions-aggregate-common@55.2.0 X +datafusion-functions-nested@55.2.0 X +datafusion-functions-table@55.2.0 X +datafusion-functions-window@55.2.0 X +datafusion-functions-window-common@55.2.0 X +datafusion-macros@55.2.0 X +datafusion-optimizer@55.2.0 X +datafusion-physical-expr@55.2.0 X +datafusion-physical-expr-adapter@55.2.0 X +datafusion-physical-expr-common@55.2.0 X +datafusion-physical-optimizer@55.2.0 X +datafusion-physical-plan@55.2.0 X +datafusion-proto@55.2.0 X +datafusion-proto-common@55.2.0 X +datafusion-proto-models@55.2.0 X +datafusion-pruning@55.2.0 X +datafusion-session@55.2.0 X +datafusion-spark@55.2.0 X +datafusion-sql@55.2.0 X datasketches@0.2.0 X der@0.7.10 X X deranged@0.5.8 X X @@ -239,10 +240,10 @@ include-flate-compress@0.3.4 X indexmap@2.14.0 X X inout@0.1.4 X X inout@0.2.2 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X itertools@0.14.0 X X +itertools@0.15.0 X X itoa@1.0.18 X X jieba-macros@0.10.3 X jieba-rs@0.10.3 X @@ -302,6 +303,7 @@ no_std_io2@0.9.4 X X nom@7.1.3 X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-bigint-dig@0.8.6 X X num-complex@0.4.6 X X num-conv@0.2.2 X X @@ -331,19 +333,18 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X paimon@0.6.0 X paimon-datafusion@0.6.0 X paimon-ftindex-core@0.1.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-vindex-core@0.5.0 X parking_lot@0.12.5 X X parking_lot_core@0.9.12 X X -parquet@58.4.0 X +parquet@59.3.2 X paste@1.0.15 X X pbkdf2@0.12.2 X X pem@4.0.0 X @@ -380,11 +381,11 @@ prost-derive@0.13.5 X prost-derive@0.14.4 X prost-types@0.14.4 X psm@0.1.31 X X -pyo3@0.28.3 X X -pyo3-build-config@0.28.3 X X -pyo3-ffi@0.28.3 X X -pyo3-macros@0.28.3 X X -pyo3-macros-backend@0.28.3 X X +pyo3@0.29.3 X X +pyo3-build-config@0.29.3 X X +pyo3-ffi@0.29.3 X X +pyo3-macros@0.29.3 X X +pyo3-macros-backend@0.29.3 X X pypaimon_rust@0.6.0 X quad-rand@0.2.3 X quick-xml@0.41.0 X @@ -493,7 +494,7 @@ strum@0.27.2 X strum_macros@0.27.2 X subtle@2.6.1 X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X @@ -514,7 +515,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X diff --git a/bindings/python/README.md b/bindings/python/README.md index fd549a086..a00c6c534 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -280,6 +280,12 @@ the ending snapshot, including when no selected files remain. The existing ### Custom Blob URI sources +Batch and stream writers can be transferred between Python threads. Operations +remain exclusive: calling another writer method while a write, prepare, or close +is running raises `RuntimeError`. Call `close()` in a `finally` block to discard +unprepared output and release source streams. Callback reference cycles are also +visible to Python's garbage collector when collection runs on another thread. + Before the first Arrow write, call `writer.with_blob_uri_reader_factory(factory)` to copy referenced Blob payloads from custom sources. The factory provides `create(uri)`, returning a reader with diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 131404e9b..b94b89c3b 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -64,7 +64,7 @@ dev = [ "maturin>=1.9.4,<2.0", "pytest>=8.0", "pyarrow>=17.0,<24.0", - "datafusion==54.0.0", + "datafusion==55.0.0", "av>=17.0,<18.0", "pillow>=12.0,<13.0", ] diff --git a/bindings/python/src/blob_uri_reader.rs b/bindings/python/src/blob_uri_reader.rs index c5cd2f913..aae49eb8d 100644 --- a/bindings/python/src/blob_uri_reader.rs +++ b/bindings/python/src/blob_uri_reader.rs @@ -20,9 +20,11 @@ use bytes::Bytes; use paimon::io::{UriInputStream, UriReader, UriReaderFactory}; -use pyo3::exceptions::PyTypeError; +use pyo3::exceptions::{PyRuntimeError, PyTypeError}; use pyo3::prelude::*; +use pyo3::{PyTraverseError, PyVisit}; use std::collections::HashMap; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, Weak}; struct PythonCallbackError(PyErr); @@ -81,8 +83,132 @@ pub(crate) fn to_callback_error(error: paimon::Error) -> PyErr { crate::error::to_py_err(error) } +struct PythonUriReference { + value: Mutex>>, + stream: bool, +} + +impl PythonUriReference { + fn new( + value: Py, + stream: bool, + references: Option<&Arc>, + ) -> Arc { + let reference = Arc::new(Self { + value: Mutex::new(Some(value)), + stream, + }); + if let Some(references) = references { + references.register(reference.clone()); + } + reference + } + + fn clone_ref(&self, py: Python<'_>) -> PyResult> { + self.value + .lock() + .unwrap() + .as_ref() + .map(|value| value.clone_ref(py)) + .ok_or_else(|| PyRuntimeError::new_err("URI callback has been cleared or closed")) + } + + fn take(&self) -> Option> { + self.value.lock().unwrap().take() + } + + fn close(&self) -> PyResult<()> { + match self.take() { + Some(stream) => tokio::task::block_in_place(move || { + Python::attach(move |py| { + stream.bind(py).call_method0("close")?; + Ok(()) + }) + }), + None => Ok(()), + } + } +} + +/// Expose every Python reference retained by a writer's URI adapters to GC. +/// The registry owns slots strongly, so traversal never upgrades and drops the +/// last owner of a Python reference while attaching to Python is forbidden. +#[derive(Default)] +pub(crate) struct PythonUriReferences { + references: Mutex>>, + prune_at: AtomicUsize, +} + +impl PythonUriReferences { + fn register(&self, reference: Arc) { + let prune = { + let mut references = self.references.lock().unwrap(); + references.push(reference); + references.len() >= self.prune_at.load(Ordering::Relaxed) + }; + if prune { + self.prune(); + } + } + + pub(crate) fn traverse(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + for reference in self.references.lock().unwrap().iter() { + // Each slot owns one Py reference, including when several slots + // happen to point at the same Python object. + visit.call(&*reference.value.lock().unwrap())?; + } + Ok(()) + } + + pub(crate) fn prune(&self) { + let retired = { + let mut references = self.references.lock().unwrap(); + let (retained, retired): (Vec<_>, Vec<_>) = std::mem::take(&mut *references) + .into_iter() + .partition(|reference| Arc::strong_count(reference) > 1); + // Bound retired readers/streams within one large write without + // rescanning every live partition's slots for each new reference. + self.prune_at.store( + retained.len().saturating_mul(2).saturating_add(16), + Ordering::Relaxed, + ); + *references = retained; + retired + }; + // Python finalizers may re-enter Native or the registry itself. + drop(retired); + } + + pub(crate) fn clear(&self) { + let references = std::mem::take(&mut *self.references.lock().unwrap()); + // Take every slot before any Python finalizer or stream close runs. + // A core close/Drop racing this clear shares the same take operation. + let values = references + .iter() + .filter_map(|reference| reference.take().map(|value| (reference.stream, value))) + .collect::>(); + drop(references); + // Cancellation may run on a runtime thread. Close callbacks and Python + // finalizers can call Native again, just like ordinary source callbacks. + tokio::task::block_in_place(move || { + Python::attach(move |py| { + for (stream, value) in values { + if stream { + // Match the cancellation Drop path: cleanup errors must + // not replace the error which caused the writer to clear. + let _ = value.bind(py).call_method0("close"); + } + // Drop Python refs inside the suspended runtime context. + drop(value); + } + }); + }); + } +} + pub(crate) struct PythonUriReaderFactory { - factory: Py, + factory: Arc, + references: Option>, // Preserve object identity across interleaved physical Blob writers without // retaining readers after core releases them. URI caching belongs to the factory. readers: Mutex>>, @@ -102,13 +228,30 @@ pub(crate) fn optional_uri_reader_factory( impl PythonUriReaderFactory { pub(crate) fn new(factory: Py, py: Python<'_>) -> PyResult { + Self::with_references(factory, py, None) + } + + pub(crate) fn tracked( + factory: Py, + py: Python<'_>, + references: Arc, + ) -> PyResult { + Self::with_references(factory, py, Some(references)) + } + + fn with_references( + factory: Py, + py: Python<'_>, + references: Option>, + ) -> PyResult { if !factory.bind(py).getattr("create")?.is_callable() { return Err(PyTypeError::new_err( "URI reader factory must provide callable create", )); } Ok(Self { - factory, + factory: PythonUriReference::new(factory, false, references.as_ref()), + references, readers: Mutex::new(HashMap::new()), }) } @@ -118,7 +261,8 @@ impl UriReaderFactory for PythonUriReaderFactory { fn supports_uri(&self, uri: &str) -> paimon::Result { tokio::task::block_in_place(|| { Python::attach(|py| { - let factory = self.factory.bind(py); + let factory = self.factory.clone_ref(py)?; + let factory = factory.bind(py); if factory.hasattr("_supports_uri")? { factory.call_method1("_supports_uri", (uri,))?.extract() } else { @@ -132,11 +276,8 @@ impl UriReaderFactory for PythonUriReaderFactory { fn create(&self, uri: &str) -> paimon::Result> { tokio::task::block_in_place(|| { Python::attach(|py| { - let reader = self - .factory - .bind(py) - .call_method1("create", (uri,))? - .unbind(); + let factory = self.factory.clone_ref(py)?; + let reader = factory.bind(py).call_method1("create", (uri,))?.unbind(); if !reader.bind(py).getattr("new_input_stream")?.is_callable() { return Err(PyTypeError::new_err( "URI reader must provide callable new_input_stream", @@ -148,8 +289,17 @@ impl UriReaderFactory for PythonUriReaderFactory { return Ok(previous as Arc); } cached.retain(|_, reader| reader.strong_count() != 0); - let reader = Arc::new(PythonUriReader { reader }); + let reader = Arc::new(PythonUriReader { + reader: PythonUriReference::new(reader, false, None), + references: self.references.clone(), + }); cached.insert(identity, Arc::downgrade(&reader)); + // Registration may prune retired slots and run Python + // finalizers. Release the identity cache before that happens. + drop(cached); + if let Some(references) = &self.references { + references.register(reader.reader.clone()); + } Ok(reader as Arc) }) }) @@ -158,7 +308,8 @@ impl UriReaderFactory for PythonUriReaderFactory { } struct PythonUriReader { - reader: Py, + reader: Arc, + references: Option>, } #[async_trait::async_trait] @@ -166,13 +317,13 @@ impl UriReader for PythonUriReader { async fn new_input_stream(&self, uri: &str) -> paimon::Result> { tokio::task::block_in_place(|| { Python::attach(|py| { - let stream = self - .reader + let reader = self.reader.clone_ref(py)?; + let stream = reader .bind(py) .call_method1("new_input_stream", (uri,))? .unbind(); Ok(Box::new(PythonUriInputStream { - stream: Some(stream), + stream: PythonUriReference::new(stream, true, self.references.as_ref()), }) as Box) }) }) @@ -181,7 +332,7 @@ impl UriReader for PythonUriReader { } struct PythonUriInputStream { - stream: Option>, + stream: Arc, } #[async_trait::async_trait] @@ -189,10 +340,8 @@ impl UriInputStream for PythonUriInputStream { async fn read(&mut self, length: usize) -> paimon::Result { tokio::task::block_in_place(|| { Python::attach(|py| { - let bytes = self - .stream - .as_ref() - .expect("open stream") + let stream = self.stream.clone_ref(py)?; + let bytes = stream .bind(py) .call_method1("read", (length,))? .extract::>()?; @@ -205,11 +354,8 @@ impl UriInputStream for PythonUriInputStream { async fn seek(&mut self, offset: u64) -> paimon::Result<()> { tokio::task::block_in_place(|| { Python::attach(|py| { - self.stream - .as_ref() - .expect("open stream") - .bind(py) - .call_method1("seek", (offset,))?; + let stream = self.stream.clone_ref(py)?; + stream.bind(py).call_method1("seek", (offset,))?; Ok(()) }) }) @@ -217,16 +363,7 @@ impl UriInputStream for PythonUriInputStream { } async fn close(&mut self) -> paimon::Result<()> { - match self.stream.take() { - Some(stream) => tokio::task::block_in_place(|| { - Python::attach(|py| { - stream.bind(py).call_method0("close")?; - Ok(()) - }) - }) - .map_err(callback_error), - None => Ok(()), - } + self.stream.close().map_err(callback_error) } } @@ -234,10 +371,183 @@ impl Drop for PythonUriInputStream { fn drop(&mut self) { // Async close is explicit in core. On cancellation, still release a // Python stream without replacing the triggering error. - if let Some(stream) = self.stream.take() { - Python::attach(|py| { - let _ = stream.bind(py).call_method0("close"); - }); - } + let _ = self.stream.close(); + } +} + +#[cfg(test)] +mod tests { + use pyo3::ffi::c_str; + use pyo3::types::PyDict; + + use super::*; + + #[pyfunction] + fn reenter_native(py: Python<'_>) -> usize { + py.detach(|| paimon_datafusion::runtime::runtime().block_on(async { 1 })) + } + + #[test] + fn cancellation_cleanup_can_reenter_native_from_a_runtime_thread() { + Python::attach(|py| { + for clear in [false, true] { + let locals = PyDict::new(py); + locals + .set_item("native", wrap_pyfunction!(reenter_native, py).unwrap()) + .unwrap(); + py.run( + c_str!( + r#" +calls = [] +class Stream: + def close(self): + calls.append(native()) +stream = Stream() +"# + ), + Some(&locals), + Some(&locals), + ) + .unwrap(); + let references = Arc::new(PythonUriReferences::default()); + let stream = PythonUriInputStream { + stream: PythonUriReference::new( + locals.get_item("stream").unwrap().unwrap().unbind(), + true, + Some(&references), + ), + }; + py.detach(|| { + paimon_datafusion::runtime::runtime().block_on(async { + if clear { + references.clear(); + } + drop(stream); + }) + }); + assert_eq!( + locals + .get_item("calls") + .unwrap() + .unwrap() + .extract::>() + .unwrap(), + vec![1], + "close must complete its Native callback, clear={clear}" + ); + } + }); + } + + #[test] + fn registry_retires_only_slots_released_by_core() { + Python::attach(|py| { + for tracked in [false, true] { + let references = Arc::new(PythonUriReferences::default()); + let object = py + .eval(c_str!("type('Callback', (), {})()"), None, None) + .unwrap(); + let weak = py + .import("weakref") + .unwrap() + .getattr("ref") + .unwrap() + .call1((&object,)) + .unwrap(); + let slot = + PythonUriReference::new(object.unbind(), false, tracked.then_some(&references)); + references.prune(); + assert!(!weak.call0().unwrap().is_none()); + drop(slot); + assert_eq!(weak.call0().unwrap().is_none(), !tracked); + references.prune(); + assert!(weak.call0().unwrap().is_none()); + } + }); + } + + #[test] + fn registration_bounds_retired_readers_within_one_write() { + Python::attach(|py| { + let references = Arc::new(PythonUriReferences::default()); + let weakref = py.import("weakref").unwrap().getattr("ref").unwrap(); + let mut readers = Vec::new(); + for _ in 0..100 { + let object = py + .eval(c_str!("type('Reader', (), {})()"), None, None) + .unwrap(); + readers.push(weakref.call1((&object,)).unwrap()); + let slot = PythonUriReference::new(object.unbind(), false, Some(&references)); + drop(slot); + } + assert!( + readers + .iter() + .filter(|reader| !reader.call0().unwrap().is_none()) + .count() + < 32, + "a large write must not retain every retired Python reader" + ); + references.prune(); + assert!(readers + .iter() + .all(|reader| reader.call0().unwrap().is_none())); + }); + } + + #[test] + fn registry_clear_and_stream_close_drop_share_one_cleanup() { + Python::attach(|py| { + for clear_first in [false, true] { + let locals = PyDict::new(py); + py.run( + c_str!( + r#" +calls = [] +class Stream: + def close(self): + calls.append('close') + raise ValueError('cleanup failed') +stream = Stream() +"# + ), + Some(&locals), + Some(&locals), + ) + .unwrap(); + let references = Arc::new(PythonUriReferences::default()); + let slot = PythonUriReference::new( + locals.get_item("stream").unwrap().unwrap().unbind(), + true, + Some(&references), + ); + let stream = PythonUriInputStream { + stream: slot.clone(), + }; + if clear_first { + references.clear(); + } else { + assert!(slot + .close() + .unwrap_err() + .is_instance_of::(py)); + } + references.clear(); + slot.close().unwrap(); + drop(stream); + references.prune(); + assert_eq!( + locals + .get_item("calls") + .unwrap() + .unwrap() + .extract::>() + .unwrap(), + vec!["close"] + ); + assert!(slot.clone_ref(py).is_err()); + assert!(references.references.lock().unwrap().is_empty()); + } + }); } } diff --git a/bindings/python/src/context.rs b/bindings/python/src/context.rs index 10a952fd6..aa11040c6 100644 --- a/bindings/python/src/context.rs +++ b/bindings/python/src/context.rs @@ -357,11 +357,11 @@ impl PaimonCatalog { py: Python<'py>, session: Bound<'py, PyAny>, ) -> PyResult> { - let name = cr"datafusion_catalog_provider".into(); + let name = cr"datafusion_catalog_provider"; let provider = Arc::clone(&self.provider) as Arc; let codec = ffi_logical_codec_from_pycapsule(session)?; let provider = FFI_CatalogProvider::new_with_ffi_codec(provider, Some(runtime()), codec); - PyCapsule::new(py, provider, Some(name)) + PyCapsule::new_with_value(py, provider, name) } /// List all databases in this catalog. diff --git a/bindings/python/src/write.rs b/bindings/python/src/write.rs index a22a15451..f89e9e345 100644 --- a/bindings/python/src/write.rs +++ b/bindings/python/src/write.rs @@ -35,7 +35,10 @@ use pyo3::prelude::*; use pyo3::types::{PyBytes, PyDict, PyString}; use pyo3::{PyTraverseError, PyVisit}; -use crate::blob_uri_reader::{callback_error, optional_uri_reader_factory, to_callback_error}; +use crate::blob_uri_reader::{ + callback_error, optional_uri_reader_factory, to_callback_error, PythonUriReaderFactory, + PythonUriReferences, +}; use crate::commit_callback::PyCommitCallbackContext; use crate::error::to_py_err; use crate::predicate::{dict_to_table_predicate, py_to_datum}; @@ -154,12 +157,13 @@ impl WriteContext { None => builder, }; Ok(WriteState { - inner: Some(WriteTarget::Table(Box::new( + inner: Mutex::new(Some(WriteTarget::Table(Box::new( builder.new_write().map_err(to_py_err)?, - ))), + )))), table_location: self.table.location().to_string(), commit_user: self.commit_user.clone(), blob_consumers: Vec::new(), + uri_references: None, }) } } @@ -559,12 +563,13 @@ impl PyPostponeFixedBucketWriteBuilder { } Ok(PyBatchTableWrite { state: WriteState { - inner: Some(WriteTarget::PostponeFixed(Box::new( + inner: Mutex::new(Some(WriteTarget::PostponeFixed(Box::new( builder.new_write().map_err(to_py_err)?, - ))), + )))), table_location: context.table.location().to_string(), commit_user: context.commit_user.clone(), blob_consumers: Vec::new(), + uri_references: None, }, prepared: false, }) @@ -576,12 +581,15 @@ impl PyPostponeFixedBucketWriteBuilder { } struct WriteState { - inner: Option, + // Encoders are Send but not Sync. PyO3's exclusive borrow protects methods; + // get_mut keeps that access lock-free while allowing GC/drop on any thread. + inner: Mutex>, table_location: String, commit_user: String, // Core keeps prior consumers in captured partition/bucket settings. Expose // every owned Python reference to GC, including replaced consumers. blob_consumers: Vec>, + uri_references: Option>, } /// Descriptor serialization and exception transport only; record boundaries, @@ -974,17 +982,37 @@ impl UpdateContext { } impl WriteState { + fn writer_mut(&mut self) -> PyResult<&mut WriteTarget> { + self.inner + .get_mut() + .unwrap() + .as_mut() + .ok_or_else(|| PyRuntimeError::new_err("TableWrite is closed")) + } + fn with_blob_uri_reader_factory( &mut self, factory: Option>, py: Python<'_>, ) -> PyResult<()> { - let factory = optional_uri_reader_factory(factory, py)?; - self.inner - .as_mut() - .ok_or_else(|| PyRuntimeError::new_err("TableWrite is closed"))? - .with_blob_uri_reader_factory(factory) - .map_err(to_py_err) + let factory = factory + .map(|factory| { + let references = self + .uri_references + .get_or_insert_with(Default::default) + .clone(); + PythonUriReaderFactory::tracked(factory, py, references) + .map(|factory| Arc::new(factory) as Arc) + }) + .transpose()?; + let result = self.writer_mut().and_then(|writer| { + writer + .with_blob_uri_reader_factory(factory) + .map_err(to_py_err) + }); + // A rejected replacement must release its newly registered Python refs too. + self.prune_callbacks(); + result } fn with_blob_consumer(&mut self, callback: Option>, py: Python<'_>) -> PyResult<()> { @@ -998,9 +1026,7 @@ impl WriteState { })) }) .transpose()?; - self.inner - .as_mut() - .ok_or_else(|| PyRuntimeError::new_err("TableWrite is closed"))? + self.writer_mut()? .with_blob_consumer( consumer .clone() @@ -1009,18 +1035,21 @@ impl WriteState { .map_err(to_py_err)?; // Release replaced callbacks which no bucket has captured. A snapshot // retained by core contributes another Arc, without another Py ref. - self.prune_blob_consumers(); + self.prune_callbacks(); if let Some(consumer) = consumer { self.blob_consumers.push(consumer); } Ok(()) } - fn prune_blob_consumers(&mut self) { + fn prune_callbacks(&mut self) { let (retained, retired): (Vec<_>, Vec<_>) = std::mem::take(&mut self.blob_consumers) .into_iter() .partition(|consumer| Arc::strong_count(consumer) > 1); self.blob_consumers = retained; + if let Some(references) = &self.uri_references { + references.prune(); + } // Python destructors can re-enter Native. No callback mutex is held, // and the retained references are restored before any destructor runs. drop(retired); @@ -1030,6 +1059,9 @@ impl WriteState { for consumer in &self.blob_consumers { visit.call(&*consumer.callback.lock().unwrap())?; } + if let Some(references) = &self.uri_references { + references.traverse(visit)?; + } Ok(()) } @@ -1039,6 +1071,9 @@ impl WriteState { .iter() .map(|consumer| consumer.callback.lock().unwrap().take()) .collect::>(); + if let Some(references) = &self.uri_references { + references.clear(); + } // Destructors may re-enter Native. Drop Python references outside locks. drop(references); } @@ -1048,9 +1083,7 @@ impl WriteState { } fn with_write_type(&mut self, columns: Vec) -> PyResult<()> { - self.inner - .as_mut() - .ok_or_else(|| PyRuntimeError::new_err("TableWrite is closed"))? + self.writer_mut()? .with_write_type(columns) .map_err(to_py_err) } @@ -1062,11 +1095,9 @@ impl WriteState { bucket: Option, ) -> PyResult<()> { let batch = crate::arrow_input::write_batch_from_pyarrow(batch)?; - let inner = self - .inner - .as_mut() - .ok_or_else(|| PyRuntimeError::new_err("TableWrite is closed"))?; + let inner = self.writer_mut()?; let result = py.detach(|| runtime().block_on(inner.write_arrow(&batch, bucket))); + self.prune_callbacks(); self.write_result(result) } @@ -1075,12 +1106,9 @@ impl WriteState { py: Python<'_>, commit_identifier: i64, ) -> PyResult> { - let inner = self - .inner - .as_mut() - .ok_or_else(|| PyRuntimeError::new_err("TableWrite is closed"))?; + let inner = self.writer_mut()?; let result = py.detach(|| runtime().block_on(inner.prepare_commit(commit_identifier))); - self.prune_blob_consumers(); + self.prune_callbacks(); let messages = result.map_err(to_callback_error)?; Ok(wrap_messages( messages, @@ -1090,11 +1118,7 @@ impl WriteState { } } -#[pyclass( - name = "BatchTableWrite", - module = "pypaimon_rust.datafusion", - unsendable -)] +#[pyclass(name = "BatchTableWrite", module = "pypaimon_rust.datafusion")] pub struct PyBatchTableWrite { state: WriteState, prepared: bool, @@ -1452,7 +1476,7 @@ impl PyBatchTableWrite { } fn close(&mut self, py: Python<'_>) { - if let Some(mut writer) = self.state.inner.take() { + if let Some(mut writer) = self.state.inner.get_mut().unwrap().take() { py.detach(|| runtime().block_on(writer.close())); } self.state.clear_callbacks(); @@ -1479,11 +1503,7 @@ impl PyBatchTableWrite { } } -#[pyclass( - name = "StreamTableWrite", - module = "pypaimon_rust.datafusion", - unsendable -)] +#[pyclass(name = "StreamTableWrite", module = "pypaimon_rust.datafusion")] pub struct PyStreamTableWrite { state: WriteState, } @@ -1527,7 +1547,7 @@ impl PyStreamTableWrite { } fn close(&mut self, py: Python<'_>) { - if let Some(mut writer) = self.state.inner.take() { + if let Some(mut writer) = self.state.inner.get_mut().unwrap().take() { py.detach(|| runtime().block_on(writer.close())); } self.state.clear_callbacks(); @@ -2428,14 +2448,14 @@ writer.with_blob_consumer(Callback(writer, 'first')) let object = locals.get_item("writer").unwrap().unwrap(); if batch { let mut writer = object.extract::>().unwrap(); - let inner = writer.state.inner.as_mut().unwrap(); + let inner = writer.state.writer_mut().unwrap(); py.detach(|| runtime().block_on(inner.write_arrow(&input, None))) .unwrap(); } else { let mut writer = object .extract::>() .unwrap(); - let inner = writer.state.inner.as_mut().unwrap(); + let inner = writer.state.writer_mut().unwrap(); py.detach(|| runtime().block_on(inner.write_arrow(&input, None))) .unwrap(); } @@ -2522,7 +2542,7 @@ first = install('first') ]) .unwrap(); let mut borrowed = writer.borrow_mut(py); - let inner = borrowed.state.inner.as_mut().unwrap(); + let inner = borrowed.state.writer_mut().unwrap(); py.detach(|| runtime().block_on(inner.write_arrow(&input, None))) .unwrap(); }; diff --git a/bindings/python/tests/test_blob_consumer.py b/bindings/python/tests/test_blob_consumer.py index d86a2ee82..aa4cf5617 100644 --- a/bindings/python/tests/test_blob_consumer.py +++ b/bindings/python/tests/test_blob_consumer.py @@ -15,7 +15,11 @@ # specific language governing permissions and limitations # under the License. +import gc import struct +import sys +import threading +import weakref from urllib.parse import urlparse import pyarrow as pa @@ -131,3 +135,100 @@ def test_blob_consumer_replacement_and_clear(tmp_path): writer.write_arrow(_batch()) finally: writer.close() + + +@pytest.mark.parametrize('stream', [False, True]) +@pytest.mark.parametrize('captured', [False, True]) +def test_writer_callback_cycles_are_collected_on_another_thread( + tmp_path, monkeypatch, stream, captured): + _, table = _table(tmp_path) + finalized = [] + unraisable = [] + monkeypatch.setattr(sys, 'unraisablehook', + lambda event: unraisable.append(str(event.exc_value))) + + class Callback: + def __init__(self, writer, label): + self.writer = writer + self.label = label + + def __call__(self, field, descriptor): + return False + + def __del__(self): + finalized.append((self.label, threading.get_ident())) + + def cycle(): + builder = (table.new_stream_write_builder() if stream + else table.new_batch_write_builder()) + writer = builder.new_write() + callback = Callback(writer, 'first') + writer.with_blob_consumer(callback) + references = [weakref.ref(callback)] + if captured: + writer.write_arrow(_batch()) + replacement = Callback(writer, 'second') + writer.with_blob_consumer(replacement) + references.append(weakref.ref(replacement)) + return references + + gc.collect() + enabled = gc.isenabled() + gc.disable() + try: + references = cycle() + collector = threading.Thread(target=gc.collect) + collector.start() + collector.join() + assert unraisable == [] + assert all(reference() is None for reference in references) + expected = ['first', 'second'] if captured else ['first'] + assert sorted(finalized) == [(label, collector.ident) for label in expected] + finally: + # Also clean up the original-thread-only implementation on failure. + gc.collect() + if enabled: + gc.enable() + + +@pytest.mark.parametrize('stream', [False, True]) +def test_writer_thread_handoff_keeps_exclusive_access(tmp_path, stream): + context, table = _table(tmp_path) + builder = (table.new_stream_write_builder() if stream + else table.new_batch_write_builder()) + writer = builder.new_write() + entered = threading.Event() + release = threading.Event() + failures = [] + + def callback(field, descriptor): + entered.set() + assert release.wait(10) + return False + + def write(): + try: + writer.write_arrow(_batch()) + except BaseException as error: + failures.append(str(error)) + + writer.with_blob_consumer(callback) + worker = threading.Thread(target=write) + try: + worker.start() + assert entered.wait(10), failures + with pytest.raises(RuntimeError, match='Already borrowed'): + writer.close() + finally: + release.set() + worker.join() + try: + assert failures == [] + messages = writer.prepare_commit(True, 1) if stream else writer.prepare_commit() + commit = builder.new_commit() + commit.commit(1, messages) if stream else commit.commit(messages) + assert pa.Table.from_batches(context.sql( + 'SELECT id, payload FROM paimon.consumer.t ORDER BY id')).to_pydict() == { + 'id': [1, 2, 3], 'payload': [b'payload', None, b'']} + finally: + writer.close() diff --git a/bindings/python/tests/test_blob_uri_reader.py b/bindings/python/tests/test_blob_uri_reader.py index aa457e225..9509bab9c 100644 --- a/bindings/python/tests/test_blob_uri_reader.py +++ b/bindings/python/tests/test_blob_uri_reader.py @@ -15,8 +15,12 @@ # specific language governing permissions and limitations # under the License. +import gc import io import struct +import sys +import threading +import weakref import pyarrow as pa import pytest @@ -108,6 +112,134 @@ def test_custom_uri_read_error_preserves_original_exception_and_closes_once(tmp_ assert not list(tmp_path.rglob('*.blob')) +@pytest.mark.parametrize('stream', [False, True]) +@pytest.mark.parametrize('source', ['unused', 'shared_reader', 'separate_reader']) +def test_uri_callback_cycles_release_every_reference_and_close_once( + tmp_path, monkeypatch, stream, source): + context, table = _table(tmp_path) + references = [] + closes = [] + unraisable = [] + monkeypatch.setattr(sys, 'unraisablehook', + lambda event: unraisable.append(str(event.exc_value))) + + class Stream: + def __init__(self): + self.buffer = io.BytesIO(b'0123456789') + + def read(self, length): + return self.buffer.read(length) + + def seek(self, offset): + return self.buffer.seek(offset) + + def close(self): + closes.append(threading.get_ident()) + # Cleanup callbacks may call Native without nesting its runtime. + context.sql('SELECT 1') + self.buffer.close() + + class Reader: + def __init__(self, writer): + self.writer = writer + references.append(weakref.ref(self)) + + def new_input_stream(self, uri): + value = Stream() + value.writer = self.writer + references.append(weakref.ref(value)) + return value + + class Factory(Reader): + def create(self, uri): + return Reader(self.writer) if source == 'separate_reader' else self + + def cycle(): + builder = (table.new_stream_write_builder() if stream + else table.new_batch_write_builder()) + writer = builder.new_write() + factory = Factory(writer) + writer.with_blob_uri_reader_factory(factory) + if source != 'unused': + writer.write_arrow(_batch([1], [descriptor(0, 3)])) + + gc.collect() + enabled = gc.isenabled() + gc.disable() + try: + cycle() + collector = threading.Thread(target=gc.collect) + collector.start() + collector.join() + assert unraisable == [] + assert all(reference() is None for reference in references) + assert closes == ([] if source == 'unused' else [collector.ident]) + finally: + for reference in references: + value = reference() + if value is not None: + value.writer.close() + gc.collect() + if enabled: + gc.enable() + + +@pytest.mark.parametrize('stream', [False, True]) +def test_uri_references_retire_after_prepare_and_rejected_replacement(tmp_path, stream): + _, table = _table(tmp_path) + builder = (table.new_stream_write_builder() if stream + else table.new_batch_write_builder()) + writer = builder.new_write() + readers = [] + streams = [] + + class Reader(ReaderFactory): + def new_input_stream(self, uri): + value = super().new_input_stream(uri) + streams.append(weakref.ref(value)) + return value + + class Factory: + def create(self, uri): + reader = Reader() + readers.append(weakref.ref(reader)) + return reader + + factory = Factory() + writer.with_blob_uri_reader_factory(factory) + try: + replacement = Factory() + retired = weakref.ref(factory) + del factory + writer.with_blob_uri_reader_factory(replacement) + assert retired() is None + retired = weakref.ref(replacement) + del replacement + writer.with_blob_uri_reader_factory(None) + assert retired() is None + writer.with_blob_uri_reader_factory(Factory()) + writer.write_arrow(_batch([1], [descriptor(0, 3)])) + assert readers[0]() is not None + assert streams[0]() is not None + rejected = Factory() + retired = weakref.ref(rejected) + with pytest.raises(ValueError, match='before any write'): + writer.with_blob_uri_reader_factory(rejected) + del rejected + assert retired() is None + writer.prepare_commit(True, 1) if stream else writer.prepare_commit() + gc.collect() + assert all(reference() is None for reference in readers + streams) + finally: + writer.close() + rejected = Factory() + retired = weakref.ref(rejected) + with pytest.raises(RuntimeError, match='closed'): + writer.with_blob_uri_reader_factory(rejected) + del rejected + assert retired() is None + + def _update_table(tmp_path): context = SQLContext() context.register_catalog('paimon', {'warehouse': str(tmp_path)}) diff --git a/crates/integration_tests/DEPENDENCIES.rust.tsv b/crates/integration_tests/DEPENDENCIES.rust.tsv index a54283d15..c3b8cd87f 100644 --- a/crates/integration_tests/DEPENDENCIES.rust.tsv +++ b/crates/integration_tests/DEPENDENCIES.rust.tsv @@ -11,20 +11,20 @@ anyhow@1.0.104 X X apache-avro@0.21.0 X approx@0.5.1 X arc-swap@1.9.2 X X -arrow@58.4.0 X -arrow-arith@58.4.0 X -arrow-array@58.4.0 X X -arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X -arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-row@58.4.0 X -arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X +arrow-array@59.3.2 X X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-row@59.3.2 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-compression@0.4.42 X X async-stream@0.3.6 X async-stream-impl@0.3.6 X @@ -165,7 +165,6 @@ include-flate@0.3.4 X include-flate-codegen@0.3.4 X include-flate-compress@0.3.4 X indexmap@2.14.0 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X itertools@0.14.0 X X @@ -226,6 +225,7 @@ no_std_io2@0.9.4 X X nom@7.1.3 X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-complex@0.4.6 X X num-conv@0.2.2 X X num-integer@0.1.46 X X @@ -243,17 +243,16 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X paimon@0.6.0 X paimon-ftindex-core@0.1.0 X paimon-integration-tests@0.6.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-vindex-core@0.5.0 X -parquet@58.4.0 X +parquet@59.3.2 X paste@1.0.15 X X percent-encoding@2.3.2 X X phf@0.12.1 X @@ -355,7 +354,7 @@ strum@0.27.2 X strum_macros@0.27.2 X subtle@2.6.1 X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X @@ -375,7 +374,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X diff --git a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv index 9e016eab7..773ee4247 100644 --- a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv +++ b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv @@ -6,6 +6,7 @@ aho-corasick@1.1.4 X X alloc-no-stdlib@2.0.4 X alloc-stdlib@0.2.4 X allocator-api2@0.2.21 X X +alp@0.0.4 X android_system_properties@0.1.5 X X anyhow@1.0.104 X X apache-avro@0.21.0 X @@ -15,20 +16,20 @@ arc-swap@1.9.2 X X arcref@0.2.0 X arrayref@0.3.9 X arrayvec@0.7.8 X X -arrow@58.4.0 X -arrow-arith@58.4.0 X -arrow-array@58.4.0 X X -arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X -arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-row@58.4.0 X -arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X +arrow-array@59.3.2 X X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-row@59.3.2 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-channel@2.5.0 X X async-compression@0.4.42 X X async-executor@1.14.0 X X @@ -54,7 +55,7 @@ base64@0.23.1 X X better_io@0.2.0 X bigdecimal@0.4.10 X X bindgen@0.72.1 X -bit-vec@0.9.1 X X +bit-vec@0.10.1 X X bitflags@2.13.1 X X bitpacking@0.9.3 X bitvec@1.1.1 X @@ -97,7 +98,6 @@ constant_time_eq@0.4.2 X X X core-foundation@0.10.1 X X core-foundation@0.9.4 X X core-foundation-sys@0.8.7 X X -core_detect@1.0.0 X X cpufeatures@0.2.17 X X cpufeatures@0.3.0 X X crc32fast@1.5.0 X X @@ -117,38 +117,38 @@ darling_core@0.23.0 X darling_macro@0.23.0 X dary_heap@0.3.9 X X dashmap@6.2.1 X -datafusion@54.0.0 X -datafusion-catalog@54.0.0 X -datafusion-catalog-listing@54.0.0 X -datafusion-common@54.0.0 X -datafusion-common-runtime@54.0.0 X -datafusion-datasource@54.0.0 X -datafusion-datasource-arrow@54.0.0 X -datafusion-datasource-csv@54.0.0 X -datafusion-datasource-json@54.0.0 X -datafusion-datasource-parquet@54.0.0 X -datafusion-doc@54.0.0 X -datafusion-execution@54.0.0 X -datafusion-expr@54.0.0 X -datafusion-expr-common@54.0.0 X -datafusion-functions@54.0.0 X -datafusion-functions-aggregate@54.0.0 X -datafusion-functions-aggregate-common@54.0.0 X -datafusion-functions-nested@54.0.0 X -datafusion-functions-table@54.0.0 X -datafusion-functions-window@54.0.0 X -datafusion-functions-window-common@54.0.0 X -datafusion-macros@54.0.0 X -datafusion-optimizer@54.0.0 X -datafusion-physical-expr@54.0.0 X -datafusion-physical-expr-adapter@54.0.0 X -datafusion-physical-expr-common@54.0.0 X -datafusion-physical-optimizer@54.0.0 X -datafusion-physical-plan@54.0.0 X -datafusion-pruning@54.0.0 X -datafusion-session@54.0.0 X -datafusion-spark@54.0.0 X -datafusion-sql@54.0.0 X +datafusion@55.2.0 X +datafusion-catalog@55.2.0 X +datafusion-catalog-listing@55.2.0 X +datafusion-common@55.2.0 X +datafusion-common-runtime@55.2.0 X +datafusion-datasource@55.2.0 X +datafusion-datasource-arrow@55.2.0 X +datafusion-datasource-csv@55.2.0 X +datafusion-datasource-json@55.2.0 X +datafusion-datasource-parquet@55.2.0 X +datafusion-doc@55.2.0 X +datafusion-execution@55.2.0 X +datafusion-expr@55.2.0 X +datafusion-expr-common@55.2.0 X +datafusion-functions@55.2.0 X +datafusion-functions-aggregate@55.2.0 X +datafusion-functions-aggregate-common@55.2.0 X +datafusion-functions-nested@55.2.0 X +datafusion-functions-table@55.2.0 X +datafusion-functions-window@55.2.0 X +datafusion-functions-window-common@55.2.0 X +datafusion-macros@55.2.0 X +datafusion-optimizer@55.2.0 X +datafusion-physical-expr@55.2.0 X +datafusion-physical-expr-adapter@55.2.0 X +datafusion-physical-expr-common@55.2.0 X +datafusion-physical-optimizer@55.2.0 X +datafusion-physical-plan@55.2.0 X +datafusion-pruning@55.2.0 X +datafusion-session@55.2.0 X +datafusion-spark@55.2.0 X +datafusion-sql@55.2.0 X datasketches@0.2.0 X deranged@0.5.8 X X diff@0.1.13 X X @@ -173,8 +173,11 @@ ext-trait-proc_macros@1.0.1 X X X extension-traits@1.0.1 X X X fallible-streaming-iterator@0.1.9 X X fastdivide@0.4.2 X X -fastlanes@0.5.2 X +fastlanes@0.6.1 X +fastlanes@0.7.2 X fastrand@2.5.0 X X +fearless_simd@1.2.0 X X +fearless_simd_macros@0.1.0 X X find-msvc-tools@0.1.9 X X fixedbitset@0.5.7 X X flatbuffers@25.12.19 X @@ -187,7 +190,7 @@ foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs4@0.13.1 X X fs_extra@1.3.0 X -fsst-rs@0.5.11 X +fsst-rs@0.6.0 X funty@2.0.0 X futures@0.3.33 X X futures-channel@0.3.33 X X @@ -245,11 +248,11 @@ include-flate@0.3.4 X include-flate-codegen@0.3.4 X include-flate-compress@0.3.4 X indexmap@2.14.0 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X itertools@0.13.0 X X itertools@0.14.0 X X +itertools@0.15.0 X X itoa@1.0.18 X X jieba-macros@0.10.3 X jieba-rs@0.10.3 X @@ -319,6 +322,7 @@ nougat@0.2.4 X X X nougat-proc_macros@0.2.4 X X X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-complex@0.4.6 X X num-conv@0.2.2 X X num-integer@0.1.46 X X @@ -331,7 +335,7 @@ object@0.37.3 X X object_store@0.13.2 X X once_cell@1.21.4 X X oneshot@0.1.13 X X -oneshot@0.2.1 X X +onpair@0.2.1 X opendal-core@0.59.5 X opendal-http-transport-reqwest@0.59.5 X opendal-layer-retry@0.59.5 X @@ -341,21 +345,24 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X paimon@0.6.0 X paimon-datafusion@0.6.0 X paimon-ftindex-core@0.1.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-vindex-core@0.5.0 X parking@2.2.1 X X parking_lot@0.12.5 X X parking_lot_core@0.9.12 X X -parquet@58.4.0 X +parquet@59.3.2 X +parquet-variant@59.3.2 X +parquet-variant-compute@59.3.2 X +parquet-variant-json@59.3.2 X paste@1.0.15 X X +pastey@0.2.3 X X pco@1.0.2 X percent-encoding@2.3.2 X X petgraph@0.8.3 X X @@ -483,7 +490,7 @@ strum_macros@0.27.2 X subtle@2.6.1 X syn@1.0.109 X X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X @@ -506,7 +513,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X @@ -542,36 +548,41 @@ utf8_iter@1.0.4 X X uuid@1.24.0 X X vcpkg@0.2.15 X X version_check@0.9.5 X X -vortex@0.75.0 X -vortex-alp@0.75.0 X -vortex-array@0.75.0 X -vortex-array-macros@0.75.0 X -vortex-btrblocks@0.75.0 X -vortex-buffer@0.75.0 X -vortex-bytebool@0.75.0 X -vortex-compressor@0.75.0 X -vortex-datetime-parts@0.75.0 X -vortex-decimal-byte-parts@0.75.0 X -vortex-error@0.75.0 X -vortex-fastlanes@0.75.0 X -vortex-file@0.75.0 X -vortex-flatbuffers@0.75.0 X -vortex-fsst@0.75.0 X -vortex-io@0.75.0 X -vortex-ipc@0.75.0 X -vortex-layout@0.75.0 X -vortex-mask@0.75.0 X -vortex-metrics@0.75.0 X -vortex-pco@0.75.0 X -vortex-proto@0.75.0 X -vortex-runend@0.75.0 X -vortex-scan@0.75.0 X -vortex-sequence@0.75.0 X -vortex-session@0.75.0 X -vortex-sparse@0.75.0 X -vortex-utils@0.75.0 X -vortex-zigzag@0.75.0 X -vortex-zstd@0.75.0 X +vortex@0.90.0 X +vortex-alp@0.90.0 X +vortex-array@0.90.0 X +vortex-array-macros@0.90.0 X +vortex-arrow@0.90.0 X +vortex-btrblocks@0.90.0 X +vortex-buffer@0.90.0 X +vortex-bytebool@0.90.0 X +vortex-compressor@0.90.0 X +vortex-compute@0.90.0 X +vortex-datetime-parts@0.90.0 X +vortex-decimal-byte-parts@0.90.0 X +vortex-edition@0.90.0 X +vortex-error@0.90.0 X +vortex-fastlanes@0.90.0 X +vortex-file@0.90.0 X +vortex-fsst@0.90.0 X +vortex-io@0.90.0 X +vortex-ipc@0.90.0 X +vortex-json@0.90.0 X +vortex-layout@0.90.0 X +vortex-mask@0.90.0 X +vortex-metrics@0.90.0 X +vortex-onpair@0.90.0 X +vortex-parquet-variant@0.90.0 X +vortex-pco@0.90.0 X +vortex-runend@0.90.0 X +vortex-scan@0.90.0 X +vortex-sequence@0.90.0 X +vortex-session@0.90.0 X +vortex-sparse@0.90.0 X +vortex-utils@0.90.0 X +vortex-uuid@0.90.0 X +vortex-zigzag@0.90.0 X +vortex-zstd@0.90.0 X walkdir@2.5.0 X X want@0.3.1 X wasi@0.11.1+wasi-snapshot-preview1 X X X diff --git a/crates/integrations/datafusion/src/blob_descriptor_functions.rs b/crates/integrations/datafusion/src/blob_descriptor_functions.rs index a49fffb27..a36ce630e 100644 --- a/crates/integrations/datafusion/src/blob_descriptor_functions.rs +++ b/crates/integrations/datafusion/src/blob_descriptor_functions.rs @@ -58,6 +58,10 @@ impl PathToDescriptorFunc { } impl ScalarUDFImpl for PathToDescriptorFunc { + fn is_strict(&self) -> bool { + true + } + fn name(&self) -> &str { PATH_TO_DESCRIPTOR } @@ -104,6 +108,10 @@ impl DescriptorToStringFunc { } impl ScalarUDFImpl for DescriptorToStringFunc { + fn is_strict(&self) -> bool { + true + } + fn name(&self) -> &str { DESCRIPTOR_TO_STRING } diff --git a/crates/integrations/datafusion/src/blob_view.rs b/crates/integrations/datafusion/src/blob_view.rs index b2fed25fe..598e619f8 100644 --- a/crates/integrations/datafusion/src/blob_view.rs +++ b/crates/integrations/datafusion/src/blob_view.rs @@ -120,6 +120,10 @@ impl Hash for BlobViewFunc { } impl ScalarUDFImpl for BlobViewFunc { + fn is_strict(&self) -> bool { + true + } + fn name(&self) -> &str { FUNCTION_NAME } diff --git a/crates/integrations/datafusion/src/filter_pushdown.rs b/crates/integrations/datafusion/src/filter_pushdown.rs index cdbe6460d..e13d797f8 100644 --- a/crates/integrations/datafusion/src/filter_pushdown.rs +++ b/crates/integrations/datafusion/src/filter_pushdown.rs @@ -249,6 +249,9 @@ impl<'a> FilterTranslator<'a> { let field = self.resolve_field(column_expr)?; let scalar = extract_scalar_literal(literal_expr)?; let datum = scalar_to_datum(scalar, field.data_type())?; + if is_float_zero(&datum) { + return None; + } match op { Operator::Eq => self.predicate_builder.equal(field.name(), datum).ok(), @@ -281,6 +284,9 @@ impl<'a> FilterTranslator<'a> { }) .collect(); let literals = literals?; + if literals.iter().any(is_float_zero) { + return None; + } self.exact(if in_list.negated { self.predicate_builder @@ -301,6 +307,9 @@ impl<'a> FilterTranslator<'a> { extract_scalar_literal(between.high.as_ref())?, field.data_type(), )?; + if is_float_zero(&low) || is_float_zero(&high) { + return None; + } // Native Between / NotBetween leaf: lets the planner / b-tree // recognize the range as a single op (see `btree::query::extract_between`). @@ -637,6 +646,19 @@ fn reverse_comparison_operator(op: Operator) -> Option { } } +/// Scalar comparisons in DataFusion 55 normalize signed zeroes, while Paimon's +/// partition and row predicates distinguish them. Reject zero boundaries instead +/// of dropping the other zero before DataFusion's residual can evaluate it. +/// Array membership still follows Arrow's total ordering, so it must retain its +/// own literal conversion without this restriction. +pub(crate) fn is_float_zero(datum: &Datum) -> bool { + match datum { + Datum::Float(value) => *value == 0.0, + Datum::Double(value) => *value == 0.0, + _ => false, + } +} + pub(crate) fn scalar_to_datum(scalar: &ScalarValue, data_type: &DataType) -> Option { match data_type { DataType::Boolean(_) => match scalar { @@ -773,8 +795,8 @@ mod tests { use paimon::catalog::Identifier; use paimon::io::FileIOBuilder; use paimon::spec::{ - ArrayType, BigIntType, FloatType, IntType, LocalZonedTimestampType, PredicateOperator, - Schema, SmallIntType, TableSchema, TimeType, TimestampType, VarCharType, + ArrayType, BigIntType, DoubleType, FloatType, IntType, LocalZonedTimestampType, + PredicateOperator, Schema, SmallIntType, TableSchema, TimeType, TimestampType, VarCharType, }; use paimon::table::Table; @@ -835,6 +857,93 @@ mod tests { } } + #[test] + fn test_float_zero_filters_fall_open_without_disabling_array_membership() { + use datafusion::functions_nested::expr_fn::array_has; + + for (data_type, zeroes) in [ + ( + DataType::Float(FloatType::new()), + [lit(0.0_f32), lit(-0.0_f32)], + ), + ( + DataType::Double(DoubleType::new()), + [lit(0.0_f64), lit(-0.0_f64)], + ), + ] { + let fields = vec![ + DataField::new(0, "id".into(), DataType::Int(IntType::new())), + DataField::new(1, "value".into(), data_type.clone()), + DataField::new( + 2, + "items".into(), + DataType::Array(ArrayType::new(data_type)), + ), + ]; + let column = Expr::Column(Column::from_name("value")); + let id_filter = Expr::Column(Column::from_name("id")).eq(lit(1)); + for zero in zeroes { + let mut filters = [ + Operator::Eq, + Operator::NotEq, + Operator::Lt, + Operator::LtEq, + Operator::Gt, + Operator::GtEq, + ] + .map(|op| { + Expr::BinaryExpr(BinaryExpr::new( + Box::new(column.clone()), + op, + Box::new(zero.clone()), + )) + }) + .to_vec(); + for negated in [false, true] { + filters.push(Expr::InList(InList::new( + Box::new(column.clone()), + vec![zero.clone()], + negated, + ))); + filters.push(Expr::Between(Between::new( + Box::new(column.clone()), + negated, + Box::new(zero.clone()), + Box::new(zero.clone()), + ))); + } + for filter in filters { + for filter in [filter.clone(), Expr::Not(Box::new(filter))] { + assert_eq!( + classify_filter_pushdown(&filter, &fields, true, |_| true), + TableProviderFilterPushDown::Unsupported, + "zero comparison must retain DataFusion semantics: {filter}" + ); + let conjunction = analyze_filters( + &[filter.clone().and(id_filter.clone())], + &fields, + true, + ); + assert!(conjunction.requires_residual); + assert_eq!( + conjunction.pushed_predicate, + build_pushed_predicate(std::slice::from_ref(&id_filter), &fields) + ); + assert!( + build_pushed_predicate(&[filter.or(id_filter.clone())], &fields) + .is_none() + ); + } + } + assert!(build_pushed_predicate( + &[array_has(Expr::Column(Column::from_name("items")), zero)], + &fields, + ) + .is_some()); + } + } + } + #[test] fn test_translate_datafusion_array_membership_functions() { use datafusion::functions_nested::expr_fn::{ diff --git a/crates/integrations/datafusion/src/lateral_vector_search.rs b/crates/integrations/datafusion/src/lateral_vector_search.rs index 2f5bea5dd..1ebb71836 100644 --- a/crates/integrations/datafusion/src/lateral_vector_search.rs +++ b/crates/integrations/datafusion/src/lateral_vector_search.rs @@ -34,14 +34,16 @@ use datafusion::arrow::array::{ use datafusion::arrow::compute::cast; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::catalog::default_table_source::source_as_provider; +use datafusion::catalog::Session; use datafusion::common::stats::Precision; -use datafusion::common::tree_node::Transformed; +use datafusion::common::tree_node::{Transformed, TreeNodeRecursion}; use datafusion::common::{ internal_err, DFSchemaRef, DataFusionError, JoinType, Result as DFResult, Statistics, }; use datafusion::datasource::TableProvider; -use datafusion::execution::context::{QueryPlanner, SessionState}; +use datafusion::execution::context::QueryPlanner; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; +use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext; use datafusion::logical_expr::utils::{conjunction, split_conjunction}; use datafusion::logical_expr::{ Expr, Extension, Filter, LogicalPlan, Projection, TableScan, UserDefinedLogicalNode, @@ -51,8 +53,8 @@ use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning, - PlanProperties, RecordBatchStream, + ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, + Partitioning, PlanProperties, RecordBatchStream, ReplaceChildrenOptions, StatisticsArgs, }; use datafusion::physical_planner::{DefaultPhysicalPlanner, ExtensionPlanner, PhysicalPlanner}; use datafusion::prelude::SessionConfig; @@ -80,7 +82,7 @@ impl QueryPlanner for PaimonQueryPlanner { async fn create_physical_plan( &self, logical_plan: &LogicalPlan, - session_state: &SessionState, + session_state: &dyn Session, ) -> DFResult> { let planner = DefaultPhysicalPlanner::with_extension_planners(vec![ Arc::new(LateralVectorSearchExtensionPlanner), @@ -418,7 +420,8 @@ impl ExtensionPlanner for LateralVectorSearchExtensionPlanner { node: &dyn UserDefinedLogicalNode, logical_inputs: &[&LogicalPlan], physical_inputs: &[Arc], - session_state: &SessionState, + session_state: &dyn Session, + planning_ctx: &PhysicalPlanningContext, ) -> DFResult>> { let Some(node) = node.as_any().downcast_ref::() else { return Ok(None); @@ -431,6 +434,7 @@ impl ExtensionPlanner for LateralVectorSearchExtensionPlanner { node.query_vector_expr(), logical_inputs[0].schema(), session_state, + planning_ctx, )?; let mut exec = LateralVectorSearchExec::new( Arc::clone(&physical_inputs[0]), @@ -475,7 +479,7 @@ struct ExecutionPreparedFilterEntry { struct ExecutionPreparedFilterCache { // DataFusion passes the same TaskContext Arc to every partition of one // execution. Keep the prepared filter alive for that TaskContext so - // sequential partitions resolve the same target snapshot. DataFusion 54 + // sequential partitions resolve the same target snapshot. DataFusion // exposes no TaskContext drop hook, so one cache-level reaper observes the // weak contexts and removes subset executions after their context dies. entries: Mutex>, @@ -857,9 +861,27 @@ impl ExecutionPlan for LateralVectorSearchExec { vec![&self.input] } + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + datafusion::physical_plan::apply_expression_roots([&self.query_vector_expr], f) + } + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DFResult> { + self.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + } + + fn replace_children( self: Arc, mut children: Vec>, + _options: ReplaceChildrenOptions, ) -> DFResult> { if children.len() != 1 { return internal_err!("LateralVectorSearchExec expects one child"); @@ -920,7 +942,11 @@ impl ExecutionPlan for LateralVectorSearchExec { }) } - fn partition_statistics(&self, _partition: Option) -> DFResult> { + fn statistics_from_inputs( + &self, + _input_stats: &[Arc], + _args: &StatisticsArgs, + ) -> DFResult> { Ok(Arc::new(Statistics { num_rows: Precision::Absent, total_byte_size: Precision::Absent, diff --git a/crates/integrations/datafusion/src/partition_count_pushdown.rs b/crates/integrations/datafusion/src/partition_count_pushdown.rs index f7fc0cbc0..3a1f90f6f 100644 --- a/crates/integrations/datafusion/src/partition_count_pushdown.rs +++ b/crates/integrations/datafusion/src/partition_count_pushdown.rs @@ -55,6 +55,7 @@ use datafusion::catalog::Session; use datafusion::common::tree_node::Transformed; use datafusion::common::{ internal_err, project_schema, Column, DataFusionError, Result as DFResult, ScalarValue, + TableReference, }; use datafusion::datasource::{provider_as_source, source_as_provider, TableProvider, TableType}; use datafusion::execution::{SendableRecordBatchStream, SessionState, TaskContext}; @@ -63,13 +64,12 @@ use datafusion::functions_aggregate::count::count_udaf; use datafusion::functions_aggregate::expr_fn::{count, sum}; use datafusion::logical_expr::expr::AggregateFunction; use datafusion::logical_expr::{ - col, lit, Aggregate, Expr, LogicalPlan, LogicalPlanBuilder, TableScan, TableSource, + col, lit, Aggregate, Expr, LogicalPlan, LogicalPlanBuilder, TableScanBuilder, TableSource, }; use datafusion::optimizer::{ApplyOrder, OptimizerConfig, OptimizerRule}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::streaming::{PartitionStream, StreamingTableExec}; use datafusion::physical_plan::ExecutionPlan; -use datafusion::sql::TableReference; use futures::{stream, TryStreamExt}; use paimon::spec::{CoreOptions, DataField, Datum, Predicate}; use paimon::table::Table; @@ -191,13 +191,9 @@ fn rewrite_aggregate(aggregate: &Aggregate) -> DFResult> { }; // Only the synthetic scan changes qualifier; fallback filters still refer // to the original scan's table name. - let counts_scan = LogicalPlan::TableScan(TableScan::try_new( - qualifier.clone(), - provider_as_source(Arc::new(counts)), - None, - vec![], - None, - )?); + let counts_scan = LogicalPlan::TableScan( + TableScanBuilder::new(qualifier.clone(), provider_as_source(Arc::new(counts))).build()?, + ); let row_count = Expr::Column(Column::new(Some(qualifier.clone()), ROW_COUNT_COLUMN)); let summed = LogicalPlan::Aggregate(Aggregate::try_new( @@ -510,13 +506,12 @@ impl PartitionRowCountStream { .iter() .map(|field| source_schema.index_of(field.name())) .collect::, _>>()?; - let scan = LogicalPlan::TableScan(TableScan::try_new( - self.table_name.clone(), - Arc::clone(source), - Some(partition_indices), - self.filters.clone(), - None, - )?); + let scan = LogicalPlan::TableScan( + TableScanBuilder::new(self.table_name.clone(), Arc::clone(source)) + .with_projection(Some(partition_indices)) + .with_filters(self.filters.clone()) + .build()?, + ); let group_by: Vec = self .partition_fields @@ -538,7 +533,7 @@ impl PartitionRowCountStream { // rewrite its own fallback. self.state .query_planner() - .create_physical_plan(&counted, &self.state) + .create_physical_plan(&counted, self.state.as_ref()) .await } } diff --git a/crates/integrations/datafusion/src/physical_plan/audit_log.rs b/crates/integrations/datafusion/src/physical_plan/audit_log.rs index 50df0b1e6..6f6e98acf 100644 --- a/crates/integrations/datafusion/src/physical_plan/audit_log.rs +++ b/crates/integrations/datafusion/src/physical_plan/audit_log.rs @@ -19,10 +19,16 @@ use std::sync::Arc; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{stats::Precision, Statistics}; use datafusion::error::Result as DFResult; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_plan::{DisplayAs, ExecutionPlan, PlanProperties}; +use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_plan::metrics::MetricsSet; +use datafusion::physical_plan::{ + ChildrenPropertiesMode, DisplayAs, ExecutionPlan, PlanProperties, ReplaceChildrenOptions, + StatisticsArgs, +}; use paimon::table::AuditLogRead; use super::PaimonTableScan; @@ -52,9 +58,31 @@ impl ExecutionPlan for PaimonAuditLogScan { vec![] } + fn metrics(&self) -> Option { + self.inner.metrics() + } + + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + self.inner.apply_expressions(f) + } + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DFResult> { + self.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + } + + fn replace_children( self: Arc, _children: Vec>, + _options: ReplaceChildrenOptions, ) -> DFResult> { Ok(self) } @@ -69,8 +97,12 @@ impl ExecutionPlan for PaimonAuditLogScan { }) } - fn partition_statistics(&self, partition: Option) -> DFResult> { - let mut statistics = self.inner.partition_statistics(partition)?; + fn statistics_from_inputs( + &self, + _input_stats: &[Arc], + args: &StatisticsArgs, + ) -> DFResult> { + let mut statistics = self.inner.statistics_from_inputs(&[], args)?; Arc::make_mut(&mut statistics).num_rows = Precision::Absent; Ok(statistics) } @@ -202,7 +234,9 @@ mod tests { )); assert!(result.updated_node.is_none()); assert_eq!( - scan.partition_statistics(None).unwrap().num_rows, + scan.statistics_from_inputs(&[], &StatisticsArgs::new()) + .unwrap() + .num_rows, Precision::Absent ); } diff --git a/crates/integrations/datafusion/src/physical_plan/scan.rs b/crates/integrations/datafusion/src/physical_plan/scan.rs index e7edacbaa..80e59aa57 100644 --- a/crates/integrations/datafusion/src/physical_plan/scan.rs +++ b/crates/integrations/datafusion/src/physical_plan/scan.rs @@ -24,6 +24,7 @@ use datafusion::arrow::datatypes::{ }; use datafusion::arrow::record_batch::{RecordBatch, RecordBatchOptions}; use datafusion::common::stats::Precision; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{ColumnStatistics, ScalarValue, Statistics}; use datafusion::config::ConfigOptions; use datafusion::datasource::physical_plan::parquet::can_expr_be_pushed_down_with_schemas; @@ -46,9 +47,13 @@ use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::filter_pushdown::{ ChildPushdownResult, FilterPushdownPhase, FilterPushdownPropagation, PushedDown, }; +use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{DisplayAs, ExecutionPlan, Partitioning, PlanProperties}; -use futures::{FutureExt, StreamExt, TryStreamExt}; +use datafusion::physical_plan::{ + ChildrenPropertiesMode, DisplayAs, ExecutionPlan, Partitioning, PlanProperties, + ReplaceChildrenOptions, StatisticsArgs, +}; +use futures::{FutureExt, Stream, StreamExt, TryStreamExt}; use paimon::arrow::ReadBudget; use paimon::spec::{ CoreOptions, DataField, Datum, MergeEngine, Predicate, PredicateBuilder, PredicateOperator, @@ -57,7 +62,7 @@ use paimon::table::{ArrowRecordBatchStream, ScanStats, Table, TableRead}; use paimon::DataSplit; use crate::error::to_datafusion_error; -use crate::filter_pushdown::scalar_to_datum; +use crate::filter_pushdown::{is_float_zero, scalar_to_datum}; fn to_datafusion_batch(batch: RecordBatch, schema: &ArrowSchemaRef) -> DFResult { if batch.num_columns() != schema.fields().len() { @@ -139,6 +144,39 @@ impl paimon::arrow::RowFilter for DataFusionRowFilter { } impl paimon::arrow::RowFilterFactory for DataFusionRowFilterFactory { + fn supports_row_group_pruning(&self) -> bool { + datafusion::physical_expr::DynamicFilterTracking::classify(&self.predicate) + .contains_dynamic_filter() + } + + fn pruning_predicates(&self, file_fields: &[DataField]) -> paimon::Result> { + let snapshot = datafusion::physical_expr_common::physical_expr::snapshot_physical_expr( + Arc::clone(&self.predicate), + ) + .map_err(datafusion_row_filter_error)?; + let builder = PredicateBuilder::new(file_fields); + Ok(split_conjunction(&snapshot) + .into_iter() + .filter(|expr| { + // Parquet min/max exclude NaN, whereas DataFusion compares floats + // using Arrow's total order. A finite bound cannot rule out NaN. + !collect_columns(expr).iter().any(|column| { + file_fields.iter().any(|field| { + field.name() == column.name() + && matches!( + field.data_type(), + paimon::spec::DataType::Float(_) + | paimon::spec::DataType::Double(_) + ) + }) + }) + }) + .filter_map(|expr| { + translate_physical_predicate(expr.as_ref(), file_fields, &builder, true) + }) + .collect()) + } + fn create( &self, context: paimon::arrow::RowFilterContext<'_>, @@ -359,6 +397,9 @@ fn translate_physical_predicate( scalar_to_datum(literal.value(), field.data_type()) }) .collect::>>()?; + if literals.iter().any(is_float_zero) { + return None; + } return if in_list.negated() { predicate_builder.is_not_in(field.name(), literals).ok() } else { @@ -513,6 +554,12 @@ fn translate_physical_comparison( }) })?; + // DataFusion equates signed zeros, while the native reader orders them separately. + // Keep zero boundaries in DataFusion so native pruning cannot discard matches. + if is_float_zero(&datum) { + return None; + } + if matches!( field.data_type(), paimon::spec::DataType::Binary(_) | paimon::spec::DataType::VarBinary(_) @@ -784,6 +831,7 @@ pub struct PaimonTableScan { /// Wrapped in `Arc` to avoid deep-cloning `DataSplit` metadata in `execute()`. planned_partitions: Vec>, plan_properties: Arc, + metrics: ExecutionPlanMetricsSet, /// Optional limit hint pushed to paimon-core planning. limit: Option, /// Whether the pushed predicate is exact (no residual filtering needed). @@ -903,6 +951,7 @@ impl PaimonTableScan { pushed_predicate, planned_partitions, plan_properties, + metrics: ExecutionPlanMetricsSet::new(), limit, filter_exact, scan_stats, @@ -1089,9 +1138,15 @@ impl PaimonTableScan { )) }; + let mut stream = Box::pin(futures::stream::once(fut).try_flatten()); + let metrics = BaselineMetrics::new(&self.metrics, partition); + let measured = futures::stream::poll_fn(move |cx| { + let _timer = metrics.elapsed_compute().timer(); + metrics.record_poll(stream.as_mut().poll_next(cx)) + }); Ok(Box::pin(RecordBatchStreamAdapter::new( self.schema(), - futures::stream::once(fut).try_flatten(), + measured, ))) } } @@ -1105,13 +1160,38 @@ impl ExecutionPlan for PaimonTableScan { &self.plan_properties } + fn metrics(&self) -> Option { + Some(self.metrics.clone_inner()) + } + fn children(&self) -> Vec<&Arc> { vec![] } + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + datafusion::physical_plan::apply_expression_roots( + self.runtime_filters.iter().chain(&self.decoder_filters), + f, + ) + } + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DFResult> { + self.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + } + + fn replace_children( self: Arc, _children: Vec>, + _options: ReplaceChildrenOptions, ) -> DFResult> { Ok(self) } @@ -1192,12 +1272,16 @@ impl ExecutionPlan for PaimonTableScan { self.execute_with(partition, context, |read, splits| read.to_arrow(splits)) } - fn partition_statistics(&self, partition: Option) -> DFResult> { + fn statistics_from_inputs( + &self, + _input_stats: &[Arc], + args: &StatisticsArgs, + ) -> DFResult> { // Statistics include hidden rows and unmasked values. if self.has_query_auth_rules() { return Ok(Arc::new(Statistics::new_unknown(&self.schema()))); } - let partitions: &[Arc<[DataSplit]>] = match partition { + let partitions: &[Arc<[DataSplit]>] = match args.partition() { Some(idx) => std::slice::from_ref(&self.planned_partitions[idx]), None => &self.planned_partitions, }; @@ -1341,6 +1425,37 @@ mod tests { )])) } + #[test] + fn runtime_pruning_refreshes_incomplete_topk_thresholds() { + use paimon::arrow::RowFilterFactory; + let column: Arc = Arc::new(Column::new("id", 0)); + let dynamic = Arc::new(DynamicFilterPhysicalExpr::new( + vec![Arc::clone(&column)], + lit(true), + )); + let filter: Arc = dynamic.clone(); + let factory = DataFusionRowFilterFactory::new(filter, test_schema()); + let fields = test_read_type(); + assert!(factory.pruning_predicates(&fields).unwrap().is_empty()); + for threshold in [100, 5] { + dynamic + .update(Arc::new(BinaryExpr::new( + Arc::clone(&column), + Operator::Lt, + lit(threshold), + ))) + .unwrap(); + assert_eq!( + factory.pruning_predicates(&fields).unwrap(), + vec![PredicateBuilder::new(&fields) + .less_than("id", Datum::Int(threshold)) + .unwrap()] + ); + } + dynamic.update(lit(true)).unwrap(); + assert!(factory.pruning_predicates(&fields).unwrap().is_empty()); + } + fn test_read_type() -> Vec { vec![DataField::new( 0, @@ -1349,6 +1464,157 @@ mod tests { )] } + #[tokio::test] + async fn runtime_row_group_pruning_preserves_nan_after_threshold_updates() { + use datafusion::arrow::array::{ArrayRef, Float32Array, Float64Array}; + use paimon::spec::{DoubleType, FloatType}; + use parquet::arrow::ArrowWriter; + use parquet::file::properties::WriterProperties; + + for single_precision in [false, true] { + for descending in [false, true] { + let dir = tempdir().unwrap(); + let bucket = dir.path().join("bucket-0"); + fs::create_dir_all(&bucket).unwrap(); + let file = bucket.join("data.parquet"); + let sign = if descending { 1.0 } else { -1.0 }; + // Floating-point arithmetic may canonicalize NaN's sign. Build + // explicit bit patterns so ascending tests really use negative NaN. + let nan = f64::from_bits(if descending { + 0x7ff8_0000_0000_0000 + } else { + 0xfff8_0000_0000_0000 + }); + let values = vec![sign * 100.0, sign * 100.0, sign, nan]; + let (data_type, array): (_, ArrayRef) = if single_precision { + ( + DataType::Float(FloatType::new()), + Arc::new(Float32Array::from( + values + .iter() + .map(|v| { + if v.is_nan() { + f32::from_bits(if descending { + 0x7fc0_0000 + } else { + 0xffc0_0000 + }) + } else { + *v as f32 + } + }) + .collect::>(), + )), + ) + } else { + ( + DataType::Double(DoubleType::new()), + Arc::new(Float64Array::from(values)), + ) + }; + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "value", + array.data_type().clone(), + true, + )])); + let batch = RecordBatch::try_new(Arc::clone(&arrow_schema), vec![array]).unwrap(); + let mut writer = ArrowWriter::try_new( + fs::File::create(&file).unwrap(), + Arc::clone(&arrow_schema), + Some( + WriterProperties::builder() + .set_max_row_group_row_count(Some(2)) + .build(), + ), + ) + .unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + let schema = PaimonSchema::builder() + .column("value", data_type) + .option("read.batch-size", "2") + .build() + .unwrap(); + let table = Table::new( + FileIOBuilder::new("file").build().unwrap(), + Identifier::new("default", "nan_topk"), + local_file_path(dir.path()), + TableSchema::new(0, &schema), + None, + ); + let split = paimon::DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path(local_file_path(&bucket)) + .with_total_buckets(1) + .with_data_files(vec![test_data_file( + "data.parquet", + 4, + fs::metadata(file).unwrap().len() as i64, + )]) + .build() + .unwrap(); + let column: Arc = Arc::new(Column::new("value", 0)); + let dynamic = Arc::new(DynamicFilterPhysicalExpr::new( + vec![Arc::clone(&column)], + lit(true), + )); + let filter: Arc = dynamic.clone(); + let read = table + .new_read_builder() + .new_read() + .unwrap() + .with_row_filter_factory(Arc::new(DataFusionRowFilterFactory::new( + filter, + arrow_schema, + ))); + let mut stream = read.to_arrow(&[split]).unwrap(); + assert_eq!(stream.try_next().await.unwrap().unwrap().num_rows(), 2); + let threshold = if single_precision { + ScalarValue::Float32(Some((sign * 100.0) as f32)) + } else { + ScalarValue::Float64(Some(sign * 100.0)) + }; + dynamic + .update(Arc::new(BinaryExpr::new( + column, + if descending { + Operator::Gt + } else { + Operator::Lt + }, + lit(threshold), + ))) + .unwrap(); + let expected = dynamic.evaluate(&batch).unwrap().into_array(4).unwrap(); + assert_eq!( + expected.as_any().downcast_ref::().unwrap(), + &BooleanArray::from(vec![false, false, false, true]) + ); + let rest = stream.try_collect::>().await.unwrap(); + assert_eq!(rest.iter().map(RecordBatch::num_rows).sum::(), 1, + "a finite Parquet bound must not discard NaN: single_precision={single_precision}, descending={descending}"); + let value = rest.iter().find(|b| b.num_rows() > 0).unwrap().column(0); + assert!(if single_precision { + value + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + .is_nan() + } else { + value + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + .is_nan() + }); + } + } + } + #[test] fn test_binary_manifest_bounds_are_not_exposed() { for data_type in [ @@ -1465,24 +1731,38 @@ mod tests { #[test] fn test_partition_statistics_include_manifest_column_bounds() { - let split = paimon::DataSplitBuilder::new() - .with_snapshot(1) - .with_partition(BinaryRow::new(0)) - .with_bucket(0) - .with_bucket_path("file:/tmp/test-table/bucket-0".to_string()) - .with_total_buckets(1) - .with_data_files(vec![ - data_file_with_int_stats("a.parquet", 4, 2, 8, 1), - data_file_with_int_stats("b.parquet", 6, 1, 10, 2), - ]) - .build() - .unwrap(); + let split = |file| { + paimon::DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path("file:/tmp/test-table/bucket-0".to_string()) + .with_total_buckets(1) + .with_data_files(vec![file]) + .build() + .unwrap() + }; let scan = PaimonTableScan::new( test_schema(), dummy_table(), test_read_type(), None, - vec![Arc::from(vec![split])], + vec![ + Arc::from(vec![split(data_file_with_int_stats( + "a.parquet", + 4, + 2, + 8, + 1, + ))]), + Arc::from(vec![split(data_file_with_int_stats( + "b.parquet", + 6, + 1, + 10, + 2, + ))]), + ], None, true, None, @@ -1490,7 +1770,8 @@ mod tests { true, ); - let statistics = scan.partition_statistics(None).unwrap(); + let context = datafusion::physical_plan::StatisticsContext::new(); + let statistics = context.compute(&scan, &StatisticsArgs::new()).unwrap(); let id = &statistics.column_statistics[0]; assert_eq!( @@ -1502,6 +1783,24 @@ mod tests { Precision::Inexact(ScalarValue::Int32(Some(10))) ); assert_eq!(id.null_count, Precision::Exact(3)); + for (partition, min, max, nulls) in [(0, 2, 8, 1), (1, 1, 10, 2)] { + let statistics = context + .compute( + &scan, + &StatisticsArgs::new().with_partition(Some(partition)), + ) + .unwrap(); + let id = &statistics.column_statistics[0]; + assert_eq!( + id.min_value, + Precision::Inexact(ScalarValue::Int32(Some(min))) + ); + assert_eq!( + id.max_value, + Precision::Inexact(ScalarValue::Int32(Some(max))) + ); + assert_eq!(id.null_count, Precision::Exact(nulls)); + } } #[test] @@ -1528,7 +1827,9 @@ mod tests { true, ); - let statistics = scan.partition_statistics(None).unwrap(); + let statistics = scan + .statistics_from_inputs(&[], &StatisticsArgs::new()) + .unwrap(); assert_eq!(statistics.num_rows, Precision::Exact(4)); assert_eq!(statistics.total_byte_size, Precision::Absent); @@ -1572,7 +1873,10 @@ mod tests { true, ); assert_eq!( - scan.partition_statistics(None).unwrap().column_statistics[0].null_count, + scan.statistics_from_inputs(&[], &StatisticsArgs::new()) + .unwrap() + .column_statistics[0] + .null_count, Precision::Absent ); } @@ -1628,7 +1932,10 @@ mod tests { true, ); - let value_stats = &scan.partition_statistics(None).unwrap().column_statistics[0]; + let value_stats = &scan + .statistics_from_inputs(&[], &StatisticsArgs::new()) + .unwrap() + .column_statistics[0]; assert_eq!(value_stats.min_value, Precision::Absent); assert_eq!(value_stats.max_value, Precision::Absent); assert_eq!(value_stats.null_count, Precision::Absent); @@ -1667,7 +1974,17 @@ mod tests { let updated = result .updated_node .expect("the scan must retain physical filters so dynamic join filters remain active"); - let statistics = updated.partition_statistics(None).unwrap(); + let mut found_dynamic_filter = false; + updated + .apply_expressions(&mut |expr| { + found_dynamic_filter |= expr.downcast_ref::().is_some(); + Ok(TreeNodeRecursion::Continue) + }) + .unwrap(); + assert!(found_dynamic_filter); + let statistics = updated + .statistics_from_inputs(&[], &StatisticsArgs::new()) + .unwrap(); assert_eq!(statistics.num_rows, Precision::Inexact(0)); assert_eq!( statistics.column_statistics[0].null_count, @@ -2095,7 +2412,14 @@ mod tests { datafusion::logical_expr::col("name"), datafusion::logical_expr::lit("ab"), ); - let physical = create_physical_expr(&logical, &df_schema, &ExecutionProps::new()).unwrap(); + let physical = create_physical_expr( + &logical, + &df_schema, + &ExecutionProps::new(), + &datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext::default( + ), + ) + .unwrap(); assert!(paimon_predicate_covers_filter( Some(&pushed), @@ -2228,6 +2552,138 @@ mod tests { )); } + #[test] + fn test_completed_dynamic_float_zero_filter_keeps_both_signed_zeros() { + use datafusion::arrow::array::{ArrayRef, Float32Array, Float64Array}; + use paimon::spec::{DoubleType, FloatType}; + + type FilterCase = (Arc, Vec>); + + for zero in [ + ScalarValue::Float32(Some(0.0)), + ScalarValue::Float32(Some(-0.0)), + ScalarValue::Float64(Some(0.0)), + ScalarValue::Float64(Some(-0.0)), + ] { + let (data_type, values): (_, ArrayRef) = match zero { + ScalarValue::Float32(_) => ( + paimon::spec::DataType::Float(FloatType::new()), + Arc::new(Float32Array::from(vec![ + Some(-1.0), + Some(-0.0), + Some(0.0), + Some(1.0), + None, + ])), + ), + _ => ( + paimon::spec::DataType::Double(DoubleType::new()), + Arc::new(Float64Array::from(vec![ + Some(-1.0), + Some(-0.0), + Some(0.0), + Some(1.0), + None, + ])), + ), + }; + let fields = vec![DataField::new(0, "value".to_string(), data_type)]; + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "value", + values.data_type().clone(), + true, + )])); + let batch = RecordBatch::try_new(Arc::clone(&schema), vec![values]).unwrap(); + let column: Arc = Arc::new(Column::new("value", 0)); + let is_negative_zero = match &zero { + ScalarValue::Float32(Some(value)) => value.is_sign_negative(), + ScalarValue::Float64(Some(value)) => value.is_sign_negative(), + _ => unreachable!(), + }; + let literal = lit(zero); + let mut cases: Vec = Vec::new(); + for (op, expected) in [ + (Operator::Eq, [false, true, true, false]), + (Operator::NotEq, [true, false, false, true]), + (Operator::Lt, [true, false, false, false]), + (Operator::LtEq, [true, true, true, false]), + (Operator::Gt, [false, false, false, true]), + (Operator::GtEq, [false, true, true, true]), + ] { + let expected = expected.into_iter().map(Some).chain([None]).collect(); + cases.push(( + Arc::new(BinaryExpr::new( + Arc::clone(&column), + op, + Arc::clone(&literal), + )), + expected, + )); + } + cases.push(( + Arc::new(BinaryExpr::new( + Arc::clone(&literal), + Operator::LtEq, + Arc::clone(&column), + )), + vec![Some(false), Some(true), Some(true), Some(true), None], + )); + // Physical IN retains total-order equality even though BinaryExpr + // comparisons normalize signed zeros. Preserve each expression's semantics. + for negated in [false, true] { + cases.push(( + Arc::new( + InListExpr::try_new( + Arc::clone(&column), + vec![Arc::clone(&literal)], + negated, + schema.as_ref(), + ) + .unwrap(), + ), + [false, is_negative_zero, !is_negative_zero, false] + .into_iter() + .map(|matched| Some(matched != negated)) + .chain([None]) + .collect(), + )); + } + cases.push(( + Arc::new(NotExpr::new(Arc::clone(&cases[0].0))), + vec![Some(true), Some(false), Some(false), Some(true), None], + )); + + for (snapshot, expected) in cases { + let dynamic = Arc::new(DynamicFilterPhysicalExpr::new( + vec![Arc::clone(&column)], + lit(true), + )); + dynamic.update(Arc::clone(&snapshot)).unwrap(); + dynamic.mark_complete(); + let dynamic: Arc = dynamic; + let plan = partition_runtime_decoder_filters(&[dynamic], &fields, true); + assert!( + plan.paimon_predicates.is_empty(), + "native total-order comparisons cannot implement {snapshot}" + ); + assert_eq!(plan.datafusion_filters.len(), 1); + let factory = DataFusionRowFilterFactory::new( + conjunction(plan.datafusion_filters), + Arc::clone(&schema), + ); + let mut filters = paimon::arrow::RowFilterFactory::create( + &factory, + paimon::arrow::RowFilterContext { + file_schema: &schema, + }, + ) + .unwrap(); + let actual = filters[0].evaluate(batch.clone()).unwrap(); + assert_eq!(actual.iter().collect::>(), expected, "{snapshot}"); + } + } + } + #[test] fn test_incomplete_dynamic_filter_stays_live_in_datafusion() { let fields = test_read_type(); diff --git a/crates/integrations/datafusion/src/physical_plan/search_score.rs b/crates/integrations/datafusion/src/physical_plan/search_score.rs index 39d01cedc..8bcbe1d2b 100644 --- a/crates/integrations/datafusion/src/physical_plan/search_score.rs +++ b/crates/integrations/datafusion/src/physical_plan/search_score.rs @@ -24,15 +24,16 @@ use datafusion::arrow::array::{ }; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::common::stats::Precision; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{internal_err, DataFusionError, Statistics}; use datafusion::error::Result as DFResult; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::EquivalenceProperties; +use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning, - PlanProperties, + ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, + Partitioning, PlanProperties, ReplaceChildrenOptions, StatisticsArgs, }; use futures::StreamExt; @@ -155,9 +156,27 @@ impl ExecutionPlan for SearchScoreExec { vec![&self.input] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + Ok(TreeNodeRecursion::Continue) + } + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DFResult> { + self.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + } + + fn replace_children( self: Arc, mut children: Vec>, + _options: ReplaceChildrenOptions, ) -> DFResult> { if children.len() != 1 { return internal_err!("SearchScoreExec expects one child"); @@ -185,7 +204,11 @@ impl ExecutionPlan for SearchScoreExec { ))) } - fn partition_statistics(&self, _partition: Option) -> DFResult> { + fn statistics_from_inputs( + &self, + _input_stats: &[Arc], + _args: &StatisticsArgs, + ) -> DFResult> { Ok(Arc::new(Statistics { num_rows: Precision::Absent, total_byte_size: Precision::Absent, diff --git a/crates/integrations/datafusion/src/selection_aggregate.rs b/crates/integrations/datafusion/src/selection_aggregate.rs index a01deeaa4..37a0f05ed 100644 --- a/crates/integrations/datafusion/src/selection_aggregate.rs +++ b/crates/integrations/datafusion/src/selection_aggregate.rs @@ -222,11 +222,18 @@ impl GroupsAccumulator for SelectionGroupsAccumulator { &mut self, values: &[ArrayRef], group_indices: &[usize], - opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> Result<()> { self.native - .merge_batch(values, group_indices, opt_filter, total_num_groups) + .merge_batch(values, group_indices, total_num_groups) + } + fn convert_to_state( + &self, + values: &[ArrayRef], + opt_filter: Option<&BooleanArray>, + ) -> Result> { + let mask = eligibility(values, self.ignore_null_values, opt_filter)?; + self.native.convert_to_state(values, mask.as_ref()) } fn size(&self) -> usize { self.native.size() + std::mem::size_of::() diff --git a/crates/integrations/datafusion/src/spark_functions.rs b/crates/integrations/datafusion/src/spark_functions.rs index 317d695b2..f727a527b 100644 --- a/crates/integrations/datafusion/src/spark_functions.rs +++ b/crates/integrations/datafusion/src/spark_functions.rs @@ -36,6 +36,9 @@ use std::sync::Arc; // Adapt input and return types while retaining upstream behavior. pub(crate) fn apply_compatibility_fixes(ctx: &SessionContext) { + // Use the compatibility rewrite for Spark's optional ignoreNulls argument + // and default RESPECT NULLS semantics. DataFusion 55's native UDAF ignores nulls. + ctx.deregister_udaf("any_value"); ctx.register_udf(date_part().as_ref().clone()); for bits in [false, true] { ctx.register_udf(ScalarUDF::from(ByteLength { @@ -297,9 +300,6 @@ impl FilteredGroups { } } impl GroupsAccumulator for FilteredGroups { - fn supports_convert_to_state(&self) -> bool { - self.0.supports_convert_to_state() - } fn convert_to_state( &self, values: &[ArrayRef], @@ -321,19 +321,8 @@ impl GroupsAccumulator for FilteredGroups { self.0.update_batch(values, groups, None, total) } } - fn merge_batch( - &mut self, - values: &[ArrayRef], - groups: &[usize], - predicate: Option<&BooleanArray>, - total: usize, - ) -> Result<()> { - if let Some(predicate) = predicate { - let (values, groups) = Self::filter(values, groups, predicate)?; - self.0.merge_batch(&values, &groups, None, total) - } else { - self.0.merge_batch(values, groups, None, total) - } + fn merge_batch(&mut self, values: &[ArrayRef], groups: &[usize], total: usize) -> Result<()> { + self.0.merge_batch(values, groups, total) } fn evaluate(&mut self, emit: EmitTo) -> Result { self.0.evaluate(emit) diff --git a/crates/integrations/datafusion/src/variant_functions.rs b/crates/integrations/datafusion/src/variant_functions.rs index c94845da9..a868447e4 100644 --- a/crates/integrations/datafusion/src/variant_functions.rs +++ b/crates/integrations/datafusion/src/variant_functions.rs @@ -55,6 +55,10 @@ impl ParseJsonFunc { } impl ScalarUDFImpl for ParseJsonFunc { + fn is_strict(&self) -> bool { + true + } + fn name(&self) -> &str { if self.try_parse { "try_parse_json" @@ -181,6 +185,10 @@ impl VariantGetFunc { } impl ScalarUDFImpl for VariantGetFunc { + fn is_strict(&self) -> bool { + true + } + fn name(&self) -> &str { if self.try_get { "try_variant_get" diff --git a/crates/integrations/datafusion/src/variant_pushdown.rs b/crates/integrations/datafusion/src/variant_pushdown.rs index 15ecfb403..d4c197a95 100644 --- a/crates/integrations/datafusion/src/variant_pushdown.rs +++ b/crates/integrations/datafusion/src/variant_pushdown.rs @@ -25,13 +25,14 @@ use std::sync::Arc; use async_trait::async_trait; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::catalog::default_table_source::source_as_provider; +use datafusion::catalog::Session; use datafusion::common::tree_node::{Transformed, TreeNode}; use datafusion::common::{ internal_err, plan_err, Column, DFSchema, DFSchemaRef, Result as DFResult, ScalarValue, }; -use datafusion::execution::context::SessionState; use datafusion::functions::core::expr_fn::get_field; use datafusion::logical_expr::expr::{InList, ScalarFunction}; +use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext; use datafusion::logical_expr::{ Between, BinaryExpr, Case, Cast, Expr, Extension, Filter, Like, LogicalPlan, Projection, TableScan, TryCast, UserDefinedLogicalNode, @@ -199,7 +200,8 @@ impl ExtensionPlanner for VariantExtractionExtensionPlanner { node: &dyn UserDefinedLogicalNode, logical_inputs: &[&LogicalPlan], physical_inputs: &[Arc], - session_state: &SessionState, + session_state: &dyn Session, + _planning_ctx: &PhysicalPlanningContext, ) -> DFResult>> { let Some(node) = node.as_any().downcast_ref::() else { return Ok(None); diff --git a/crates/integrations/datafusion/src/vector_search.rs b/crates/integrations/datafusion/src/vector_search.rs index 715ce37ee..c483d0843 100644 --- a/crates/integrations/datafusion/src/vector_search.rs +++ b/crates/integrations/datafusion/src/vector_search.rs @@ -28,17 +28,19 @@ use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::catalog::Session; use datafusion::catalog::TableFunctionImpl; use datafusion::common::stats::Precision; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{internal_err, project_schema, Statistics}; use datafusion::datasource::{TableProvider, TableType}; use datafusion::error::{DataFusionError, Result as DFResult}; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::logical_expr::{Expr, TableProviderFilterPushDown}; -use datafusion::physical_expr::EquivalenceProperties; +use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; use datafusion::physical_plan::empty::EmptyExec; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, + ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, + PlanProperties, ReplaceChildrenOptions, StatisticsArgs, }; use datafusion::prelude::SessionContext; use futures::{stream, TryStreamExt}; @@ -475,9 +477,27 @@ impl ExecutionPlan for VectorSearchExec { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + Ok(TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>, + ) -> DFResult> { + self.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + } + + fn replace_children( + self: Arc, + children: Vec>, + _options: ReplaceChildrenOptions, ) -> DFResult> { if !children.is_empty() { return internal_err!("VectorSearchExec is a leaf and takes no children"); @@ -503,7 +523,11 @@ impl ExecutionPlan for VectorSearchExec { ))) } - fn partition_statistics(&self, _partition: Option) -> DFResult> { + fn statistics_from_inputs( + &self, + _input_stats: &[Arc], + _args: &StatisticsArgs, + ) -> DFResult> { Ok(Arc::new(Statistics { num_rows: Precision::Absent, total_byte_size: Precision::Absent, diff --git a/crates/integrations/datafusion/tests/count_pushdown.rs b/crates/integrations/datafusion/tests/count_pushdown.rs index fd800618c..f76420eaf 100644 --- a/crates/integrations/datafusion/tests/count_pushdown.rs +++ b/crates/integrations/datafusion/tests/count_pushdown.rs @@ -19,7 +19,7 @@ //! //! DataFusion's `aggregate_statistics` optimizer rule can replace a //! scan+aggregate subtree with a literal projection when the source -//! reports `Precision::Exact(num_rows)` from `partition_statistics()`. +//! reports `Precision::Exact(num_rows)` from `statistics_from_inputs()`. //! //! These tests verify that: //! - COUNT(*) pushdown succeeds when statistics are exact (no PaimonTableScan in plan) diff --git a/crates/integrations/datafusion/tests/datafusion55_features.rs b/crates/integrations/datafusion/tests/datafusion55_features.rs new file mode 100644 index 000000000..33049cbd9 --- /dev/null +++ b/crates/integrations/datafusion/tests/datafusion55_features.rs @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod common; + +use datafusion::common::JoinType; +use datafusion::execution::TaskContext; +use datafusion::logical_expr::LogicalPlan; +use datafusion::physical_plan::collect; +use std::sync::Arc; + +fn join_type(plan: &LogicalPlan) -> Option { + if let LogicalPlan::Join(join) = plan { + return Some(join.join_type); + } + plan.inputs().into_iter().find_map(join_type) +} + +#[tokio::test] +async fn strict_udfs_enable_outer_join_elimination_without_changing_variant_null() { + let (_tmp, ctx) = common::setup_sql_context().await; + for function in [ + "path_to_descriptor(r.path)", + "descriptor_to_string(path_to_descriptor(r.path))", + "parse_json(r.json)", + "try_parse_json(r.json)", + "variant_get(parse_json(r.json), '$.a')", + "try_variant_get(parse_json(r.json), '$.a')", + "blob_view('test_db.metrics', 0, r.id)", + ] { + let sql = format!("SELECT l.id FROM (VALUES (1), (2)) l(id) LEFT JOIN (VALUES (1, 'x', '{{\"a\": 1}}')) r(id, path, json) ON l.id = r.id WHERE {function} IS NOT NULL"); + let frame = ctx.sql(&sql).await.unwrap(); + let plan = frame.into_optimized_plan().unwrap(); + assert_eq!( + join_type(&plan), + Some(JoinType::Inner), + "{function}: {plan:?}" + ); + assert_eq!(common::row_count(&ctx, &sql).await, 1); + } + let sql = "SELECT l.id FROM (VALUES (1), (2)) l(id) LEFT JOIN (VALUES (1, 'null')) r(id, json) ON l.id = r.id WHERE NOT is_variant_null(parse_json(r.json))"; + assert_eq!(common::row_count(&ctx, sql).await, 1); +} + +#[tokio::test] +async fn explain_pgjson_reports_actual_scan_output_rows() { + let (_tmp, ctx) = common::setup_sql_context().await; + common::exec(&ctx, "CREATE TABLE paimon.test_db.metrics (id INT)").await; + common::exec( + &ctx, + "INSERT INTO paimon.test_db.metrics VALUES (1), (2), (3)", + ) + .await; + for (relation, scan_name, expected_scan_rows) in [ + ("metrics", "PaimonTableScan", 2), + ("metrics$audit_log", "PaimonAuditLogScan", 2), + ] { + let query = format!("SELECT id FROM paimon.test_db.{relation} WHERE id > 1"); + let frame = ctx.sql(&query).await.unwrap(); + let plan = frame.create_physical_plan().await.unwrap(); + let batches = collect(Arc::clone(&plan), Arc::new(TaskContext::default())) + .await + .unwrap(); + assert_eq!( + batches.iter().map(|batch| batch.num_rows()).sum::(), + 2 + ); + fn scan_rows( + plan: &dyn datafusion::physical_plan::ExecutionPlan, + name: &str, + ) -> Option { + if plan.name() == name { + return plan.metrics().and_then(|metrics| metrics.output_rows()); + } + plan.children() + .into_iter() + .find_map(|child| scan_rows(child.as_ref(), name)) + } + assert_eq!( + scan_rows(plan.as_ref(), scan_name), + Some(expected_scan_rows) + ); + let output = ctx + .sql(&format!( + "EXPLAIN (ANALYZE, FORMAT pgjson, METRICS 'rows') {query}" + )) + .await + .unwrap() + .collect() + .await + .unwrap(); + let value: serde_json::Value = serde_json::from_str(common::string_value( + output[0].column_by_name("plan").unwrap().as_ref(), + 0, + )) + .unwrap(); + fn scan_node<'a>( + value: &'a serde_json::Value, + name: &str, + ) -> Option<&'a serde_json::Value> { + match value { + serde_json::Value::Object(values) => { + if values + .get("Node Type") + .and_then(|v| v.as_str()) + .is_some_and(|v| v.contains(name)) + { + return Some(value); + } + values.values().find_map(|v| scan_node(v, name)) + } + serde_json::Value::Array(values) => values.iter().find_map(|v| scan_node(v, name)), + _ => None, + } + } + let scan = scan_node(&value, scan_name).expect("Paimon scan in pgjson explain"); + assert_eq!(scan["Actual Rows"], expected_scan_rows, "{value}"); + } +} + +#[tokio::test] +async fn new_array_functions_run_on_paimon_columns() { + let (_tmp, ctx) = common::setup_sql_context().await; + common::exec( + &ctx, + "CREATE TABLE paimon.test_db.arrays (id INT, values_ ARRAY)", + ) + .await; + common::exec( + &ctx, + "INSERT INTO paimon.test_db.arrays VALUES (1, [1, 2, 3]), (2, [])", + ) + .await; + let rows = common::collect_id_value(&ctx, + "SELECT id, CAST(array_sum(array_scale(values_, 2)) AS INT) AS value FROM paimon.test_db.arrays WHERE id = 1").await; + assert_eq!(rows, vec![(1, 12)]); + assert_eq!( + common::row_count( + &ctx, + "SELECT unnest_outer(values_) FROM paimon.test_db.arrays" + ) + .await, + 4 + ); +} diff --git a/crates/integrations/datafusion/tests/format_table_statistics.rs b/crates/integrations/datafusion/tests/format_table_statistics.rs index b29ae0f1e..667fb7832 100644 --- a/crates/integrations/datafusion/tests/format_table_statistics.rs +++ b/crates/integrations/datafusion/tests/format_table_statistics.rs @@ -151,7 +151,7 @@ async fn test_empty_format_table_counts_zero() { ); } -/// The partition-pruned plan reaches `partition_statistics()` too, so a pruned +/// The partition-pruned plan reaches `statistics_from_inputs()` too, so a pruned /// `COUNT(*)` must also come from the data and not from the placeholder. /// /// Only predicated queries are asserted here. An unfiltered scan over a diff --git a/crates/integrations/datafusion/tests/remaining_compat.rs b/crates/integrations/datafusion/tests/remaining_compat.rs index 850346010..e940ab90a 100644 --- a/crates/integrations/datafusion/tests/remaining_compat.rs +++ b/crates/integrations/datafusion/tests/remaining_compat.rs @@ -98,44 +98,59 @@ async fn selection_aggregates_merge_native_state_across_batches_and_partitions() }; use datafusion::datasource::MemTable; use std::sync::Arc; - let ctx = SQLContext::new(); - let schema = Arc::new(Schema::new(vec![ - Field::new("g", DataType::Int64, false), - Field::new("v", DataType::Int64, true), - Field::new("k", DataType::Int64, true), - ])); - let batch = |g: Vec, v: Vec>, k: Vec>| { - RecordBatch::try_new( - Arc::clone(&schema), + // Exercise both ordinary partial aggregation and direct state conversion. + for bypass in [false, true] { + let ctx = SQLContext::new(); + ctx.sql("SET datafusion.execution.target_partitions = 2") + .await + .unwrap(); + if bypass { + ctx.sql( + "SET datafusion.execution.skip_partial_aggregation_probe_ratio_threshold = 0.0", + ) + .await + .unwrap(); + ctx.sql("SET datafusion.execution.skip_partial_aggregation_probe_rows_threshold = 0") + .await + .unwrap(); + } + let schema = Arc::new(Schema::new(vec![ + Field::new("g", DataType::Int64, false), + Field::new("v", DataType::Int64, true), + Field::new("k", DataType::Int64, true), + ])); + let batch = |g: Vec, v: Vec>, k: Vec>| { + RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int64Array::from(g)), + Arc::new(Int64Array::from(v)), + Arc::new(Int64Array::from(k)), + ], + ) + .unwrap() + }; + let partitions = vec![ vec![ - Arc::new(Int64Array::from(g)), - Arc::new(Int64Array::from(v)), - Arc::new(Int64Array::from(k)), + batch(vec![1, 2], vec![Some(10), Some(200)], vec![Some(1), None]), + batch(vec![1, 3], vec![None, Some(300)], vec![Some(4), Some(10)]), ], - ) - .unwrap() - }; - let partitions = vec![ - vec![ - batch(vec![1, 2], vec![Some(10), Some(200)], vec![Some(1), None]), - batch(vec![1, 3], vec![None, Some(300)], vec![Some(4), Some(10)]), - ], - vec![ - batch( - vec![1, 2, 3], - vec![Some(30), Some(201), None], - vec![Some(3), None, Some(11)], - ), - batch(vec![1, 3], vec![Some(999), Some(301)], vec![None, Some(12)]), - ], - ]; - ctx.ctx() - .register_table( - "selection_inputs", - Arc::new(MemTable::try_new(schema, partitions).unwrap()), - ) - .unwrap(); - assert_eq!( + vec![ + batch( + vec![1, 2, 3], + vec![Some(30), Some(201), None], + vec![Some(3), None, Some(11)], + ), + batch(vec![1, 3], vec![Some(999), Some(301)], vec![None, Some(12)]), + ], + ]; + ctx.ctx() + .register_table( + "selection_inputs", + Arc::new(MemTable::try_new(schema, partitions).unwrap()), + ) + .unwrap(); + assert_eq!( rows( &ctx, "SELECT g,arg_max(CAST(NULL AS BIGINT),1) FROM selection_inputs GROUP BY g ORDER BY g" @@ -143,15 +158,18 @@ async fn selection_aggregates_merge_native_state_across_batches_and_partitions() .await, vec![vec!["1", "NULL"], vec!["2", "NULL"], vec!["3", "NULL"]] ); - assert_eq!(rows(&ctx,"SELECT g,max_by(v,k),min_by(v,k),arg_max(v,k),max_by(v,k) FILTER (WHERE v<300 OR v IS NULL) FROM selection_inputs GROUP BY g ORDER BY g").await,vec![vec!["1","NULL","10","30","NULL"],vec!["2","NULL","NULL","NULL","NULL"],vec!["3","301","300","301","NULL"]]); - assert_eq!( - rows( - &ctx, - "SELECT max_by(v,k),min_by(v,k),arg_max(v,k) FROM selection_inputs" - ) - .await, - vec![vec!["301", "10", "301"]] - ); + assert_eq!(rows(&ctx,"SELECT g,max_by(v,k),min_by(v,k),arg_max(v,k),max_by(v,k) FILTER (WHERE v<300 OR v IS NULL) FROM selection_inputs GROUP BY g ORDER BY g").await,vec![vec!["1","NULL","10","30","NULL"],vec!["2","NULL","NULL","NULL","NULL"],vec!["3","301","300","301","NULL"]]); + assert_eq!( + rows( + &ctx, + "SELECT max_by(v,k),min_by(v,k),arg_max(v,k) FROM selection_inputs" + ) + .await, + vec![vec!["301", "10", "301"]] + ); + assert_eq!(rows(&ctx, "SELECT g,arg_max(v,k) FILTER (WHERE k>2) FROM selection_inputs GROUP BY g ORDER BY g").await, + vec![vec!["1", "30"],vec!["2", "NULL"],vec!["3", "301"]]); + } } #[tokio::test] @@ -229,17 +247,25 @@ async fn aggregate_rewrites_keep_filters_constant_keys_and_local_registrations() vec![vec!["a", "a", "a"]] ); assert_eq!(rows(&ctx,"SELECT array_length(collect_list(p OR q)) FROM (VALUES (NULL,FALSE),(FALSE,TRUE)) t(p,q)").await,vec![vec!["1"]]); - ctx.ctx().register_udf(create_udf( - "first", - vec![DataType::Int64], - DataType::Int64, - Volatility::Immutable, - Arc::new(|args| Ok(args[0].clone())), - )); - assert_eq!( - rows(&ctx, "SELECT first(v) FROM (VALUES (1),(2)) t(v)").await, - vec![vec!["1"], vec!["2"]] - ); + assert_eq!(rows(&ctx, "SELECT any_value(v),any_value(v,false),any_value(v,true) FROM (VALUES (CAST(NULL AS BIGINT)),(7)) t(v)").await, + vec![vec!["NULL", "NULL", "7"]]); + for name in ["first", "any_value"] { + ctx.ctx().register_udf(create_udf( + name, + vec![DataType::Int64], + DataType::Int64, + Volatility::Immutable, + Arc::new(|args| Ok(args[0].clone())), + )); + assert_eq!( + rows( + &ctx, + &format!("SELECT {name}(v) FROM (VALUES (1),(2)) t(v)") + ) + .await, + vec![vec!["1"], vec!["2"]] + ); + } } #[tokio::test] diff --git a/crates/integrations/datafusion/tests/scan_pruning_stats.rs b/crates/integrations/datafusion/tests/scan_pruning_stats.rs index 27929709a..dd721945d 100644 --- a/crates/integrations/datafusion/tests/scan_pruning_stats.rs +++ b/crates/integrations/datafusion/tests/scan_pruning_stats.rs @@ -355,9 +355,8 @@ async fn test_physical_plan_displays_scan_stats_summary() { /// files here are exactly the shapes the writer produces for the two ways the /// rule breaks: `[1.0, NaN]` reports `min == max == 1.0` over two rows because /// NaN goes into neither min/max nor the null count, and a file holding only -/// `-0.0` reports `min = -0.0, max = 0.0`, which `Datum` sees as equal to `0.0` -/// while the row-level filter does not. Pruning either file drops a row the -/// filter would have returned. +/// `-0.0` reports `min = -0.0, max = 0.0`. Keep pruning conservative for both +/// layouts and let DataFusion apply its own NaN and signed-zero semantics. #[tokio::test] async fn test_not_equal_keeps_float_files_with_nan_or_signed_zero() { let (_tmp, catalog) = common::create_test_env(); @@ -395,11 +394,11 @@ async fn test_not_equal_keeps_float_files_with_nan_or_signed_zero() { "SELECT id FROM paimon.test_db.stats_float WHERE d <> 1.0 ORDER BY id", vec![2, 3], ), - // `-0.0` is not `0.0` to the filter, so row 3 survives; rows 1 and 2 are - // kept as well because `1.0 <> 0.0` and NaN is not `0.0` either. + // DataFusion 55 treats signed zeroes as equal, matching its native + // in-memory reader; NaN and 1.0 still differ from zero. ( "SELECT id FROM paimon.test_db.stats_float WHERE d <> 0.0 ORDER BY id", - vec![1, 2, 3], + vec![1, 2], ), ( "SELECT id FROM paimon.test_db.stats_float WHERE d NOT IN (1.0, 2.0) ORDER BY id", @@ -423,3 +422,127 @@ async fn test_not_equal_keeps_float_files_with_nan_or_signed_zero() { assert_eq!(ids, expected, "wrong rows for `{sql}`"); } } + +/// DataFusion 55 compares both zero signs as equal. Paimon's own predicates +/// retain Arrow/Java total-order semantics, including when pruning partitions. +/// Compare against an unfiltered in-memory copy so an inexact pushdown cannot +/// silently remove rows before DataFusion applies its residual. +#[tokio::test] +async fn test_float_zero_pushdown_matches_native_datafusion() { + use std::sync::Arc; + + use datafusion::arrow::array::Int32Array; + use datafusion::arrow::record_batch::RecordBatch; + use datafusion::datasource::MemTable; + use datafusion::prelude::SessionContext; + + fn ids(batches: &[RecordBatch]) -> Vec { + batches + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect() + } + + let (_tmp, catalog) = common::create_test_env(); + let sql_context = common::create_sql_context(catalog).await; + common::exec(&sql_context, "CREATE SCHEMA paimon.test_db").await; + for (name, partitioning) in [ + ("zero_data", ""), + ("zero_partitions", "PARTITIONED BY (f, d)"), + ] { + common::exec( + &sql_context, + &format!( + "CREATE TABLE paimon.test_db.{name} \ + (id INT, f FLOAT, d DOUBLE, items ARRAY) {partitioning}" + ), + ) + .await; + common::exec( + &sql_context, + &format!( + "INSERT INTO paimon.test_db.{name} VALUES \ + (1, CAST(-0.0 AS FLOAT), CAST(-0.0 AS DOUBLE), [CAST(-0.0 AS DOUBLE)]), \ + (2, CAST(0.0 AS FLOAT), CAST(0.0 AS DOUBLE), [CAST(0.0 AS DOUBLE)]), \ + (3, -1.0, -1.0, [-1.0]), (4, 1.0, 1.0, [1.0]), \ + (5, CAST('NaN' AS FLOAT), CAST('NaN' AS DOUBLE), [CAST('NaN' AS DOUBLE)]), \ + (6, NULL, NULL, NULL)" + ), + ) + .await; + let batches = sql_context + .sql(&format!("SELECT * FROM paimon.test_db.{name}")) + .await + .unwrap() + .collect() + .await + .unwrap(); + let native = SessionContext::new(); + native + .register_table( + "native", + Arc::new(MemTable::try_new(batches[0].schema(), vec![batches]).unwrap()), + ) + .unwrap(); + + let mut predicates = Vec::new(); + for column in ["f", "d"] { + for zero in ["0.0", "-0.0"] { + for operator in ["=", "<>", "<", "<=", ">", ">="] { + predicates.push(format!("{column} {operator} {zero}")); + } + predicates.extend([ + format!("{zero} = {column}"), + format!("{column} IN ({zero}, 1.0)"), + format!("{column} NOT IN ({zero}, 1.0)"), + format!("{column} BETWEEN {zero} AND 1.0"), + format!("{column} BETWEEN -1.0 AND {zero}"), + format!("{column} NOT BETWEEN {zero} AND 1.0"), + format!("{column} NOT BETWEEN -1.0 AND {zero}"), + format!("NOT ({column} = {zero})"), + format!("NOT ({column} <= {zero})"), + format!("{column} = {zero} AND id > 0"), + format!("{column} = {zero} OR id = 4"), + ]); + } + } + // Array membership continues to distinguish zero signs in DataFusion 55. + for function in ["array_has", "array_has_any", "array_has_all"] { + let needle = if function == "array_has" { + "0.0" + } else { + "[0.0]" + }; + predicates.push(format!("{function}(items, {needle})")); + } + for predicate in predicates { + let actual = sql_context + .sql(&format!( + "SELECT id FROM paimon.test_db.{name} WHERE {predicate} ORDER BY id" + )) + .await + .unwrap() + .collect() + .await + .unwrap(); + let expected = native + .sql(&format!( + "SELECT id FROM native WHERE {predicate} ORDER BY id" + )) + .await + .unwrap() + .collect() + .await + .unwrap(); + assert_eq!(ids(&actual), ids(&expected), "{name}: {predicate}"); + } + } +} diff --git a/crates/paimon-rest-server/DEPENDENCIES.rust.tsv b/crates/paimon-rest-server/DEPENDENCIES.rust.tsv index 35d5429d5..bca31d1d7 100644 --- a/crates/paimon-rest-server/DEPENDENCIES.rust.tsv +++ b/crates/paimon-rest-server/DEPENDENCIES.rust.tsv @@ -11,20 +11,20 @@ anyhow@1.0.104 X X apache-avro@0.21.0 X approx@0.5.1 X arc-swap@1.9.2 X X -arrow@58.4.0 X -arrow-arith@58.4.0 X -arrow-array@58.4.0 X X -arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X -arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-row@58.4.0 X -arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X +arrow-array@59.3.2 X X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-row@59.3.2 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-compression@0.4.42 X X async-stream@0.3.6 X async-stream-impl@0.3.6 X @@ -168,7 +168,6 @@ include-flate@0.3.4 X include-flate-codegen@0.3.4 X include-flate-compress@0.3.4 X indexmap@2.14.0 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X itertools@0.14.0 X X @@ -230,6 +229,7 @@ no_std_io2@0.9.4 X X nom@7.1.3 X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-complex@0.4.6 X X num-conv@0.2.2 X X num-integer@0.1.46 X X @@ -247,17 +247,16 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X paimon@0.6.0 X paimon-ftindex-core@0.1.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-rest-server@0.6.0 X paimon-vindex-core@0.5.0 X -parquet@58.4.0 X +parquet@59.3.2 X paste@1.0.15 X X percent-encoding@2.3.2 X X phf@0.12.1 X @@ -361,7 +360,7 @@ strum@0.27.2 X strum_macros@0.27.2 X subtle@2.6.1 X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X @@ -381,7 +380,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X diff --git a/crates/paimon/Cargo.toml b/crates/paimon/Cargo.toml index 3bc4385cf..a92f02ab6 100644 --- a/crates/paimon/Cargo.toml +++ b/crates/paimon/Cargo.toml @@ -130,8 +130,8 @@ arrow-array = { workspace = true } arrow-arith = { workspace = true } arrow-buffer = { workspace = true } arrow-cast = { workspace = true } -arrow-json = "58.0" -arrow-ipc = "58.0" +arrow-json = "59.2" +arrow-ipc = "59.2" arrow-ord = { workspace = true } arrow-row = { workspace = true } arrow-schema = { workspace = true } @@ -141,7 +141,7 @@ futures = "0.3" crossbeam-channel = "0.5" tokio-util = { workspace = true, features = ["compat", "io-util"] } parquet = { workspace = true, features = ["async", "zstd", "lz4", "snap"] } -orc-rust = "0.8.0" +orc-rust = "0.9.0" async-stream = "0.3.6" reqwest = { version = "0.12", features = ["json", "gzip", "deflate"] } # Already in the tree via `reqwest`; direct dep for the object storage User-Agent transport. @@ -156,11 +156,11 @@ twox-hash = { version = "2.1.3", default-features = false, features = ["xxhash64 md-5 = "0.10" regex = "1" uuid = { version = "1", features = ["v4"] } -paimon-mosaic-core = "0.2.0" +paimon-mosaic-core = "0.3.0" paimon-ftindex-core = "0.1.0" tempfile = "3" paimon-vindex-core = "0.5.0" -vortex = { version = "0.75.0", features = ["tokio"], optional = true } +vortex = { version = "0.90.0", features = ["tokio"], optional = true } libloading = "0.9" log = "0.4" # Keep CI on the dependency set that passed before unicode-segmentation 1.13.3. diff --git a/crates/paimon/DEPENDENCIES.rust.tsv b/crates/paimon/DEPENDENCIES.rust.tsv index 4080653ae..a8a771744 100644 --- a/crates/paimon/DEPENDENCIES.rust.tsv +++ b/crates/paimon/DEPENDENCIES.rust.tsv @@ -8,26 +8,27 @@ aho-corasick@1.1.4 X X alloc-no-stdlib@2.0.4 X alloc-stdlib@0.2.4 X allocator-api2@0.2.21 X X +alp@0.0.4 X android_system_properties@0.1.5 X X anyhow@1.0.104 X X apache-avro@0.21.0 X approx@0.5.1 X arc-swap@1.9.2 X X arcref@0.2.0 X -arrow@58.4.0 X -arrow-arith@58.4.0 X -arrow-array@58.4.0 X X -arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X -arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-row@58.4.0 X -arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X +arrow-array@59.3.2 X X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-row@59.3.2 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-channel@2.5.0 X X async-compression@0.4.42 X X async-executor@1.14.0 X X @@ -54,7 +55,7 @@ base64ct@1.8.3 X X better_io@0.2.0 X bigdecimal@0.4.10 X X bindgen@0.72.1 X -bit-vec@0.9.1 X X +bit-vec@0.10.1 X X bitflags@2.13.1 X X bitpacking@0.9.3 X bitvec@1.1.1 X @@ -101,7 +102,6 @@ const_for@0.1.6 X core-foundation@0.10.1 X X core-foundation@0.9.4 X X core-foundation-sys@0.8.7 X X -core_detect@1.0.0 X X cpubits@0.1.1 X X cpufeatures@0.2.17 X X cpufeatures@0.3.0 X X @@ -153,8 +153,11 @@ ext-trait-proc_macros@1.0.1 X X X extension-traits@1.0.1 X X X fallible-streaming-iterator@0.1.9 X X fastdivide@0.4.2 X X -fastlanes@0.5.2 X +fastlanes@0.6.1 X +fastlanes@0.7.2 X fastrand@2.5.0 X X +fearless_simd@1.2.0 X X +fearless_simd_macros@0.1.0 X X find-msvc-tools@0.1.9 X X flatbuffers@25.12.19 X flate2@1.1.9 X X @@ -165,7 +168,7 @@ foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs4@0.13.1 X X fs_extra@1.3.0 X -fsst-rs@0.5.11 X +fsst-rs@0.6.0 X funty@2.0.0 X futures@0.3.33 X X futures-channel@0.3.33 X X @@ -227,11 +230,11 @@ include-flate-compress@0.3.4 X indexmap@2.14.0 X X inout@0.1.4 X X inout@0.2.2 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X itertools@0.13.0 X X itertools@0.14.0 X X +itertools@0.15.0 X X itoa@1.0.18 X X jieba-macros@0.10.3 X jieba-rs@0.10.3 X @@ -302,6 +305,7 @@ nougat@0.2.4 X X X nougat-proc_macros@0.2.4 X X X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-bigint-dig@0.8.6 X X num-complex@0.4.6 X X num-conv@0.2.2 X X @@ -315,7 +319,7 @@ objc2-core-foundation@0.3.2 X X X objc2-system-configuration@0.3.2 X X X once_cell@1.21.4 X X oneshot@0.1.13 X X -oneshot@0.2.1 X X +onpair@0.2.1 X opendal-core@0.59.5 X opendal-http-transport-reqwest@0.59.5 X opendal-layer-retry@0.59.5 X @@ -332,20 +336,23 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X paimon@0.6.0 X paimon-ftindex-core@0.1.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-vindex-core@0.5.0 X parking@2.2.1 X X parking_lot@0.12.5 X X parking_lot_core@0.9.12 X X -parquet@58.4.0 X +parquet@59.3.2 X +parquet-variant@59.3.2 X +parquet-variant-compute@59.3.2 X +parquet-variant-json@59.3.2 X paste@1.0.15 X X +pastey@0.2.3 X X pbkdf2@0.12.2 X X pco@1.0.2 X pem@4.0.0 X @@ -485,7 +492,7 @@ strum_macros@0.27.2 X subtle@2.6.1 X syn@1.0.109 X X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X @@ -508,7 +515,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X @@ -543,36 +549,41 @@ utf8_iter@1.0.4 X X uuid@1.24.0 X X vcpkg@0.2.15 X X version_check@0.9.5 X X -vortex@0.75.0 X -vortex-alp@0.75.0 X -vortex-array@0.75.0 X -vortex-array-macros@0.75.0 X -vortex-btrblocks@0.75.0 X -vortex-buffer@0.75.0 X -vortex-bytebool@0.75.0 X -vortex-compressor@0.75.0 X -vortex-datetime-parts@0.75.0 X -vortex-decimal-byte-parts@0.75.0 X -vortex-error@0.75.0 X -vortex-fastlanes@0.75.0 X -vortex-file@0.75.0 X -vortex-flatbuffers@0.75.0 X -vortex-fsst@0.75.0 X -vortex-io@0.75.0 X -vortex-ipc@0.75.0 X -vortex-layout@0.75.0 X -vortex-mask@0.75.0 X -vortex-metrics@0.75.0 X -vortex-pco@0.75.0 X -vortex-proto@0.75.0 X -vortex-runend@0.75.0 X -vortex-scan@0.75.0 X -vortex-sequence@0.75.0 X -vortex-session@0.75.0 X -vortex-sparse@0.75.0 X -vortex-utils@0.75.0 X -vortex-zigzag@0.75.0 X -vortex-zstd@0.75.0 X +vortex@0.90.0 X +vortex-alp@0.90.0 X +vortex-array@0.90.0 X +vortex-array-macros@0.90.0 X +vortex-arrow@0.90.0 X +vortex-btrblocks@0.90.0 X +vortex-buffer@0.90.0 X +vortex-bytebool@0.90.0 X +vortex-compressor@0.90.0 X +vortex-compute@0.90.0 X +vortex-datetime-parts@0.90.0 X +vortex-decimal-byte-parts@0.90.0 X +vortex-edition@0.90.0 X +vortex-error@0.90.0 X +vortex-fastlanes@0.90.0 X +vortex-file@0.90.0 X +vortex-fsst@0.90.0 X +vortex-io@0.90.0 X +vortex-ipc@0.90.0 X +vortex-json@0.90.0 X +vortex-layout@0.90.0 X +vortex-mask@0.90.0 X +vortex-metrics@0.90.0 X +vortex-onpair@0.90.0 X +vortex-parquet-variant@0.90.0 X +vortex-pco@0.90.0 X +vortex-runend@0.90.0 X +vortex-scan@0.90.0 X +vortex-sequence@0.90.0 X +vortex-session@0.90.0 X +vortex-sparse@0.90.0 X +vortex-utils@0.90.0 X +vortex-uuid@0.90.0 X +vortex-zigzag@0.90.0 X +vortex-zstd@0.90.0 X walkdir@2.5.0 X X want@0.3.1 X wasi@0.11.1+wasi-snapshot-preview1 X X X diff --git a/crates/paimon/src/arrow/format/avro.rs b/crates/paimon/src/arrow/format/avro.rs index 2b593559c..c1f1ce37c 100644 --- a/crates/paimon/src/arrow/format/avro.rs +++ b/crates/paimon/src/arrow/format/avro.rs @@ -410,14 +410,14 @@ fn build_column( DataType::Map(map_type) => build_map_column(records, name, map_type, num_rows)?, // Java encodes MULTISET as a map from the element to an INT count, // sharing the MAP path (`AvroSchemaConverter#extractValueTypeToAvroMap` - // returns IntType). Unlike MAP, `paimon_type_to_arrow` lets the key here - // follow the element's nullability and pins the count non-nullable. + // returns IntType). MAP keys and MULTISET elements are non-nullable + // in Arrow; counts are non-nullable too. DataType::Multiset(multiset_type) => build_map_like_column( records, name, multiset_type.element_type(), &DataType::Int(IntType::new()), - multiset_type.element_type().is_nullable(), + false, false, num_rows, )?, @@ -608,7 +608,7 @@ fn build_map_column( /// The key and value types are passed separately rather than taken from a /// `MapType` so `MULTISET` can reuse this with an implicit `INT` count as the /// value. The two nullability flags must match what `paimon_type_to_arrow` -/// declares for the same type -- MAP and MULTISET differ here -- or +/// declares for the same type or /// `RecordBatch::try_new` rejects the column. fn build_map_like_column( records: &[Value], diff --git a/crates/paimon/src/arrow/format/parquet.rs b/crates/paimon/src/arrow/format/parquet.rs index 0ade1fbf6..a2de1a289 100644 --- a/crates/paimon/src/arrow/format/parquet.rs +++ b/crates/paimon/src/arrow/format/parquet.rs @@ -904,7 +904,10 @@ impl FormatFileReader for ParquetFormatReader { Some(fp) => (&fp.predicates, &fp.file_fields), None => (&empty_predicates, &[]), }; - let row_filter_factory = predicates.and_then(|fp| fp.row_filter_factory.as_deref()); + let row_filter_factory = predicates.and_then(|fp| fp.row_filter_factory.clone()); + let runtime_pruning = row_filter_factory + .as_ref() + .is_some_and(|factory| factory.supports_row_group_pruning()); // Predicates need both indexes for page-stat pruning. Row selection only // needs OffsetIndex so arrow-rs can avoid fetching unselected pages. @@ -981,7 +984,7 @@ impl FormatFileReader for ParquetFormatReader { .map(ParquetRowFilter::into_predicates) .unwrap_or_default(); - if let Some(factory) = row_filter_factory { + if let Some(factory) = row_filter_factory.as_deref() { let file_schema = batch_stream_builder.schema(); match factory.create(RowFilterContext { file_schema }) { Ok(filters) => { @@ -1041,14 +1044,15 @@ impl FormatFileReader for ParquetFormatReader { // Resource-aware sequential reads build one row-group stream at a time, // so admission precedes its data I/O. Preserve stateful decoder filters // across those streams instead of recreating the engine's factory. - let shared_filters = if self.read_budget.as_ref().is_some_and(|b| b.has_resources()) { - std::mem::take(&mut decoder_predicates) - .into_iter() - .map(SharedParquetPredicate::new) - .collect::>() - } else { - Vec::new() - }; + let shared_filters = + if runtime_pruning || self.read_budget.as_ref().is_some_and(|b| b.has_resources()) { + std::mem::take(&mut decoder_predicates) + .into_iter() + .map(SharedParquetPredicate::new) + .collect::>() + } else { + Vec::new() + }; if !decoder_predicates.is_empty() { batch_stream_builder = batch_stream_builder.with_row_filter(ParquetRowFilter::new(decoder_predicates)); @@ -1105,45 +1109,46 @@ impl FormatFileReader for ParquetFormatReader { .min(batch_stream_builder.metadata().num_row_groups()) }) .unwrap_or(1); - let selected_row_groups = self - .read_budget - .as_ref() - .filter(|budget| { + let selected_row_groups = (runtime_pruning + || self.read_budget.as_ref().is_some_and(|budget| { row_group_parallelism > 1 || budget.diagnostics_enabled() || budget.has_resources() - }) - .map(|budget| { - let mut row_group_selection = combined_selection; - let selected_row_groups = batch_stream_builder - .metadata() - .row_groups() - .iter() - .enumerate() - .filter_map(|(row_group_index, row_group)| { - let selection = row_group_selection - .as_mut() - .map(|selection| selection.split_off(row_group.num_rows() as usize)); - if selection - .as_ref() - .is_some_and(|selection| !selection.selects_any()) - { - return None; - } - let projection = if budget.has_resources() { - &memory_mask - } else { - &mask - }; - let projected_bytes = projected_row_group_bytes(row_group, projection); - Some((row_group_index, selection, projected_bytes)) - }) - .collect::>(); - let projected_bytes = selected_row_groups - .iter() - .map(|(_, _, projected_bytes)| *projected_bytes) - .collect::>(); + })) + .then(|| { + let budget = self.read_budget.as_ref(); + let mut row_group_selection = combined_selection; + let selected_row_groups = batch_stream_builder + .metadata() + .row_groups() + .iter() + .enumerate() + .filter_map(|(row_group_index, row_group)| { + let selection = row_group_selection + .as_mut() + .map(|selection| selection.split_off(row_group.num_rows() as usize)); + if selection + .as_ref() + .is_some_and(|selection| !selection.selects_any()) + { + return None; + } + let projection = if budget.is_some_and(|budget| budget.has_resources()) { + &memory_mask + } else { + &mask + }; + let projected_bytes = projected_row_group_bytes(row_group, projection); + Some((row_group_index, selection, projected_bytes)) + }) + .collect::>(); + let projected_bytes = selected_row_groups + .iter() + .map(|(_, _, projected_bytes)| *projected_bytes) + .collect::>(); + if let Some(budget) = budget { budget.record_projected_row_groups(&projected_bytes); - selected_row_groups - }); + } + selected_row_groups + }); if row_group_parallelism > 1 { let selected_row_groups = selected_row_groups.expect("parallel row-group reads need a selection plan"); @@ -1243,8 +1248,12 @@ impl FormatFileReader for ParquetFormatReader { return Ok(stream.boxed()); } - if let Some(budget) = self.read_budget.as_ref().filter(|b| b.has_resources()) { - let budget = Arc::clone(budget); + let budget = self + .read_budget + .as_ref() + .filter(|budget| budget.has_resources()) + .cloned(); + if budget.is_some() || runtime_pruning { let selected = selected_row_groups.expect("resource-aware reads need a selection plan"); let metadata = ArrowReaderMetadata::try_new( batch_stream_builder.metadata().clone(), @@ -1255,12 +1264,38 @@ impl FormatFileReader for ParquetFormatReader { row_filter_factory: None, file_fields: file_fields.to_vec(), }); + let runtime_fields = file_fields.to_vec(); + let pruning_metadata = Arc::clone(batch_stream_builder.metadata()); + let column_indices = pruning_metadata + .row_groups() + .first() + .map(|group| build_row_group_column_indices(group.columns(), &runtime_fields)) + .unwrap_or_default(); + let identity_mapping = (0..runtime_fields.len()).map(Some).collect::>(); return Ok(async_stream::try_stream! { for (index, selection, estimated_bytes) in selected { + // Refresh immediately before data I/O, never while building the + // whole-file selection: a TopK threshold can tighten after a batch. + if runtime_pruning { + let factory = row_filter_factory.as_ref().expect("runtime pruning has a factory"); + let may_match = match factory.pruning_predicates(&runtime_fields) { + Ok(predicates) => predicates_may_match_with_schema( + &predicates, + &ParquetRowGroupStats { row_group: pruning_metadata.row_group(index), column_indices: &column_indices }, + &identity_mapping, + &runtime_fields, + ), + Err(error) => { + log::warn!("failed to refresh runtime Parquet pruning: {error}"); + true + } + }; + if !may_match { continue; } + } // Foreground and merge reads only try memory admission. They // never hold a shared scheduling slot while awaiting another // merge input, nor wait for output buffers owned downstream. - let _permit = budget.reserve_memory(estimated_bytes)?; + let _permit = budget.as_ref().map(|budget| budget.reserve_memory(estimated_bytes)).transpose()?; let mut stream = build_row_group_stream( Arc::clone(&shared_reader), file_size, metadata.clone(), mask.clone(), index, batch_size, selection, shared_filters.clone(), @@ -5088,10 +5123,10 @@ mod tests { assert_eq!(column.physical_type(), parquet::basic::Type::INT64); assert_eq!( column.logical_type_ref(), - Some(&parquet::basic::LogicalType::Timestamp { - is_adjusted_to_u_t_c: adjusted_to_utc, - unit: parquet::basic::TimeUnit::MILLIS, - }) + Some(&parquet::basic::LogicalType::timestamp( + adjusted_to_utc, + parquet::basic::TimeUnit::MILLIS, + )) ); let ParquetStatistics::Int64(stats) = metadata.row_group(0).column(index).statistics().unwrap() @@ -7183,6 +7218,130 @@ mod tests { assert_eq!(ids, vec![1, 2, 3, 4, 5]); } + #[tokio::test] + async fn runtime_row_group_pruning_refreshes_before_io_and_falls_open() { + #[derive(Debug)] + struct ThresholdFactory { + threshold: Arc, + calls: Arc, + fail: bool, + } + impl crate::arrow::RowFilterFactory for ThresholdFactory { + fn create( + &self, + _: crate::arrow::RowFilterContext<'_>, + ) -> crate::Result>> { + Ok(vec![]) + } + fn supports_row_group_pruning(&self) -> bool { + true + } + fn pruning_predicates(&self, fields: &[DataField]) -> crate::Result> { + self.calls.fetch_add(1, AtomicOrdering::SeqCst); + if self.fail { + return Err(Error::DataInvalid { + message: "unavailable runtime threshold".into(), + source: None, + }); + } + let limit = self.threshold.load(AtomicOrdering::SeqCst); + if limit == usize::MAX { + return Ok(vec![]); + } + Ok(vec![PredicateBuilder::new(fields) + .less_than("id", Datum::Int(limit as i32)) + .unwrap()]) + } + } + for (resources, selected_ranges) in + [(false, false), (true, false), (false, true), (true, true)] + { + let mut read_bytes = Vec::new(); + for (statistics, fail) in [ + (EnabledStatistics::Chunk, false), + (EnabledStatistics::None, false), + (EnabledStatistics::Chunk, true), + ] { + let data = + Bytes::from(write_multi_row_group_parquet(64, 512, statistics, true).await); + let tracker = TrackingFileRead::new(data.clone()); + let threshold = Arc::new(AtomicUsize::new(usize::MAX)); + let calls = Arc::new(AtomicUsize::new(0)); + let fields = vec![ + residual_test_field(0, "id", DataType::Int(IntType::new())), + residual_test_field(1, "value", DataType::Int(IntType::new())), + ]; + let predicates = FilePredicates { + predicates: vec![], + file_fields: fields.clone(), + row_filter_factory: Some(Arc::new(ThresholdFactory { + threshold: Arc::clone(&threshold), + calls: Arc::clone(&calls), + fail, + })), + }; + let reader = if resources { + ParquetFormatReader::with_read_budget(Arc::new( + ReadBudget::default().with_resources( + crate::resource::ResourceContext::builder() + .memory_limit(1024 * 1024) + .build() + .unwrap(), + ), + )) + } else { + ParquetFormatReader::default() + }; + let mut stream = reader + .read_batch_stream( + Box::new(tracker.clone()), + data.len() as u64, + &fields, + Some(&predicates), + Some(64), + selected_ranges + .then(|| vec![RowRange::new(3, 68), RowRange::new(130, 260)]), + ) + .await + .unwrap(); + tracker.reset(); + let first = stream.next().await.unwrap().unwrap(); + assert_eq!(first.num_rows(), if selected_ranges { 61 } else { 64 }); + assert_eq!( + first + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(first.num_rows() - 1), + 63 + ); + threshold.store(64, AtomicOrdering::SeqCst); + let rest = stream.try_collect::>().await.unwrap(); + let rows: usize = rest.iter().map(RecordBatch::num_rows).sum(); + assert_eq!( + rows, + if statistics == EnabledStatistics::Chunk && !fail { + 0 + } else if selected_ranges { + 136 + } else { + 448 + } + ); + assert_eq!( + calls.load(AtomicOrdering::SeqCst), + if selected_ranges { 5 } else { 8 } + ); + read_bytes.push(tracker.bytes_read()); + } + assert!( + read_bytes[0] < read_bytes[1], + "pruning must avoid data I/O: {read_bytes:?}" + ); + } + } + #[tokio::test] async fn test_external_row_filters_are_ordered_by_parquet_cost() { use std::sync::Mutex; diff --git a/crates/paimon/src/arrow/format/row.rs b/crates/paimon/src/arrow/format/row.rs index 3294fdf68..1bbc0c890 100644 --- a/crates/paimon/src/arrow/format/row.rs +++ b/crates/paimon/src/arrow/format/row.rs @@ -1406,12 +1406,7 @@ impl ColumnBuilder { DataType::Multiset(m) => { let count_type = DataType::Int(IntType::new()); Self::Map { - entries_field: map_entries_field( - m.element_type(), - &count_type, - m.element_type().is_nullable(), - false, - )?, + entries_field: map_entries_field(m.element_type(), &count_type, false, false)?, offsets: vec![0], validities: Vec::with_capacity(capacity), keys: Box::new(ColumnBuilder::new(m.element_type(), capacity)?), @@ -3452,7 +3447,7 @@ mod tests { let bag = test_map_array( vec![0, 2, 2, 3], vec![true, false, true], - true, + false, false, vec![Some("x"), Some("y"), Some("z")], vec![Some(2), Some(1), Some(4)], diff --git a/crates/paimon/src/arrow/format/vortex.rs b/crates/paimon/src/arrow/format/vortex.rs index 92fb7bf95..b4c4a3004 100644 --- a/crates/paimon/src/arrow/format/vortex.rs +++ b/crates/paimon/src/arrow/format/vortex.rs @@ -24,14 +24,13 @@ use crate::spec::{DataField, Predicate}; use crate::table::{ArrowRecordBatchStream, RowRange}; use crate::Error; use arrow_array::RecordBatch; -use arrow_schema::{DataType as ArrowDataType, Field, SchemaRef}; +use arrow_schema::{DataType as ArrowDataType, SchemaRef}; use async_trait::async_trait; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, OnceLock}; -use vortex::array::arrow::{ArrowSessionExt, FromArrowArray}; -use vortex::array::dtype::arrow::FromArrowType; use vortex::array::dtype::DType; use vortex::array::{ArrayRef, VortexSessionExecute}; +use vortex::arrow::ArrowSessionExt; use vortex::buffer::ByteBuffer; use vortex::file::{OpenOptionsSessionExt, WriteOptionsSessionExt}; use vortex::io::runtime::current::CurrentThreadRuntime; @@ -181,9 +180,9 @@ fn read_vortex_batches( .iter() .any(|field| matches!(field.data_type(), ArrowDataType::Struct(_))) { - // Vortex 0.75 can apply a child ROW's validity to its parent while - // projecting recursive struct layouts. Read complete ROWs and their - // independent validity before applying selection and outer projection. + // Preserve each ROW's independent validity when walking recursive + // layouts, including files written by older Vortex versions. Apply + // selection and outer projection after reconstructing complete ROWs. let names = scan_schema .fields() .iter() @@ -197,8 +196,10 @@ fn read_vortex_batches( unreachable!() }; let indices = arrow_array::UInt64Array::from_iter_values(rows.iter()); - let indices = - ArrayRef::from_arrow(&indices, false).map_err(|error| Error::DataInvalid { + let indices = session + .arrow() + .from_arrow_array(Arc::new(indices), false) + .map_err(|error| Error::DataInvalid { message: format!("Failed to build Vortex row selection: {error}"), source: None, })?; @@ -230,7 +231,14 @@ fn read_vortex_batches( // Apply column projection. { use vortex::array::expr::{root, select}; - scan_builder = scan_builder.with_projection(select(projected_names, root())); + scan_builder = scan_builder.with_projection( + select(projected_names, root()) + .bind(vortex_file.dtype()) + .map_err(|e| Error::DataInvalid { + message: format!("Failed to bind Vortex projection: {e}"), + source: None, + })?, + ); } // Vortex 0.68 filtered scans can block indefinitely on some runtimes. @@ -424,7 +432,7 @@ fn vortex_array_to_record_batch( vortex_array: ArrayRef, schema: &SchemaRef, ) -> crate::Result { - let target_field = Field::new( + let target_field = arrow_schema::Field::new( "", ArrowDataType::Struct(schema.fields().clone()), vortex_array.dtype().is_nullable(), @@ -475,6 +483,8 @@ fn vortex_array_to_record_batch( /// writes all accumulated arrays through Vortex into an in-memory buffer before /// flushing that buffer to Paimon's output file. pub(crate) struct VortexFormatWriter { + /// Reuse the Arrow conversion registry across input batches. + session: VortexSession, /// Vortex dtype derived from the target Arrow schema. dtype: DType, /// Converted arrays pending final Vortex write. @@ -490,10 +500,17 @@ pub(crate) struct VortexFormatWriter { impl VortexFormatWriter { pub(crate) async fn new(output: &OutputFile, schema: SchemaRef) -> crate::Result { validate_vortex_schema(&schema)?; - let dtype = DType::from_arrow(schema); + let session = VortexSession::default(); + let dtype = session + .arrow() + .from_arrow_schema(schema.as_ref()) + .map_err(|e| Error::Unsupported { + message: format!("Failed to convert Vortex schema: {e}"), + })?; let bytes_written = Arc::new(AtomicU64::new(0)); Ok(Self { + session, dtype, arrays: Vec::new(), output: output.clone(), @@ -503,10 +520,9 @@ impl VortexFormatWriter { } } -/// `DType::from_arrow` ends in `unimplemented!` for a type it has no arm for, so screen -/// the schema first. `Map` is the only such type Paimon builds — it is what -/// `paimon_type_to_arrow` emits for both `MAP` and `MULTISET` — and it can sit at any -/// depth, so the walk descends the same containers `DType::from_arrow` does. +/// Screen the schema before Arrow conversion. MAP and MULTISET remain +/// unsupported at any depth, so the +/// walk also checks the metadata marking the Arrow list used for MULTISET. fn validate_vortex_schema(schema: &SchemaRef) -> crate::Result<()> { for field in schema.fields() { if contains_unconvertible_type(field.data_type()) { @@ -543,8 +559,11 @@ fn contains_unconvertible_type(data_type: &ArrowDataType) -> bool { impl FormatFileWriter for VortexFormatWriter { async fn write(&mut self, batch: &RecordBatch) -> crate::Result<()> { let staged_bytes = batch.get_array_memory_size(); - let vortex_arr = - ArrayRef::from_arrow(batch.clone(), false).map_err(|e| Error::DataInvalid { + let vortex_arr = self + .session + .arrow() + .from_arrow_record_batch(batch.clone(), batch.schema_ref()) + .map_err(|e| Error::DataInvalid { message: format!("Failed to convert RecordBatch to Vortex: {e}"), source: None, })?; @@ -571,6 +590,7 @@ impl FormatFileWriter for VortexFormatWriter { async fn close(self: Box) -> crate::Result { let this = *self; let VortexFormatWriter { + session: _, dtype, arrays, output, @@ -652,6 +672,116 @@ mod tests { use futures::StreamExt; use std::ops::Range; + #[tokio::test] + async fn reads_vortex_0_75_file_with_nested_nulls_projection_and_selection() { + use crate::spec::{IntType, RowType}; + use arrow_array::{Array, StructArray}; + use futures::TryStreamExt; + + let bytes = Bytes::from_static(include_bytes!( + "../../../testdata/vortex_compat/vortex_0_75.vortex" + )); + let fields = vec![ + DataField::new(0, "id".into(), DataType::Int(IntType::new())), + DataField::new( + 1, + "name".into(), + DataType::VarChar(VarCharType::string_type()), + ), + DataField::new( + 2, + "profile".into(), + DataType::Row(RowType::new(vec![DataField::new( + 3, + "detail".into(), + DataType::Row(RowType::new(vec![DataField::new( + 4, + "keep".into(), + DataType::Int(IntType::new()), + )])), + )])), + ), + ]; + for sparse in [false, true] { + let projection = if sparse { + vec![fields[2].clone(), fields[0].clone()] + } else { + fields.clone() + }; + let selection = sparse.then(|| vec![RowRange::new(1, 2), RowRange::new(511, 511)]); + let batches = VortexFormatReader + .read_batch_stream( + Box::new(WholeFileOnlyRead { + bytes: bytes.clone(), + }), + bytes.len() as u64, + &projection, + None, + None, + selection, + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let schema = crate::arrow::build_target_arrow_schema(&projection).unwrap(); + let batch = arrow_select::concat::concat_batches(&schema, &batches).unwrap(); + let expected_ids = if sparse { + vec![1, 2, 511] + } else { + (0..512).collect() + }; + let ids = batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(ids.values().as_ref(), expected_ids); + let profile = batch + .column_by_name("profile") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let detail = profile + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let keep = detail + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for (row, id) in expected_ids.iter().enumerate() { + assert_eq!(profile.is_valid(row), id % 4 != 2); + if profile.is_valid(row) { + assert_eq!(detail.is_valid(row), id % 4 != 1); + if detail.is_valid(row) { + assert_eq!(keep.value(row), if id % 4 == 0 { 7 } else { 9 }); + } + } + if !sparse { + let name = batch + .column_by_name("name") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(name.is_null(row), id % 4 == 1); + if name.is_valid(row) { + assert_eq!( + name.value(row), + ["one", "", "three", "four"][*id as usize % 4] + ); + } + } + } + } + } + fn test_arrow_schema() -> Arc { Arc::new(ArrowSchema::new(vec![ ArrowField::new("id", ArrowDataType::Int32, false), @@ -696,9 +826,13 @@ mod tests { Some(NullBuffer::from(vec![true, true, false])), ); let original = RecordBatch::try_new(schema.clone(), vec![Arc::new(profile)]).unwrap(); + let session = VortexSession::default(); let memory = vortex_array_to_record_batch( - &VortexSession::default(), - ArrayRef::from_arrow(original.clone(), false).unwrap(), + &session, + session + .arrow() + .from_arrow_record_batch(original.clone(), original.schema_ref()) + .unwrap(), &schema, ) .unwrap(); diff --git a/crates/paimon/src/arrow/format/vortex/nested.rs b/crates/paimon/src/arrow/format/vortex/nested.rs index b19dfd2a0..a9287b59e 100644 --- a/crates/paimon/src/arrow/format/vortex/nested.rs +++ b/crates/paimon/src/arrow/format/vortex/nested.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -//! Read complete ROWs without Vortex 0.75's recursive StructReader projection. -//! Each ROW has independent validity; a null child must not null its parent. +//! Read complete ROWs with independent validity, including older Vortex layouts. +//! A null child must not null its parent during recursive field projection. use super::CurrentThreadRuntime; use crate::{Error, Result}; @@ -56,6 +56,13 @@ pub(super) fn read_array( ) } +fn child(layout: &LayoutRef, slot: usize) -> Result { + layout + .slot(slot) + .map_err(invalid)? + .ok_or_else(|| invalid(format!("missing layout child slot {slot}"))) +} + fn projected_dtype(dtype: &DType, names: Option<&[String]>) -> Result { let Some(names) = names else { return Ok(dtype.clone()); @@ -101,7 +108,7 @@ fn read_layout( runtime, session, source.clone(), - &layout.child(0).map_err(invalid)?, + &child(layout, 0)?, None, )?) } else { @@ -112,9 +119,7 @@ fn read_layout( let index = original .find(name) .ok_or_else(|| invalid(format!("missing root field {name}")))?; - let child = layout - .child(index + usize::from(nullable)) - .map_err(invalid)?; + let child = child(layout, index + 1)?; columns.push(read_layout(runtime, session, source.clone(), &child, None)?); } return StructArray::try_new( @@ -133,7 +138,7 @@ fn read_layout( runtime, session, source.clone(), - &layout.child(index).map_err(invalid)?, + &child(layout, index)?, roots, )?); } @@ -143,28 +148,19 @@ fn read_layout( } if layout.is::() { // Values use dictionary coordinates, codes use row coordinates. - let values = read_layout( - runtime, - session, - source.clone(), - &layout.child(0).map_err(invalid)?, - roots, - )?; - let codes = read_layout( - runtime, - session, - source, - &layout.child(1).map_err(invalid)?, - None, - )?; + let values = read_layout(runtime, session, source.clone(), &child(layout, 0)?, roots)?; + let codes = read_layout(runtime, session, source, &child(layout, 1)?, None)?; return DictArray::try_new(codes, values) .map(IntoArray::into_array) .map_err(invalid); } // Zone maps wrap data without changing its schema or row coordinates. - for (index, kind) in layout.child_types().enumerate() { - if matches!(kind, LayoutChildType::Transparent(_)) { - let child = layout.child(index).map_err(invalid)?; + for index in 0..layout.nslots() { + if matches!( + layout.slot_type(index), + Some(LayoutChildType::Transparent(_)) + ) { + let child = child(layout, index)?; if child.dtype() == layout.dtype() && child.row_count() == layout.row_count() { return read_layout(runtime, session, source, &child, roots); } @@ -217,11 +213,10 @@ fn read_layout( #[cfg(test)] mod tests { use super::*; - use vortex::array::dtype::arrow::FromArrowType; + use vortex::arrow::ArrowSessionExt; use vortex::io::session::RuntimeSessionExt; use vortex::layout::layouts::chunked::ChunkedLayout; use vortex::layout::segments::{SegmentFuture, SegmentId}; - use vortex::layout::IntoLayout; use vortex::layout::LayoutChildren; use vortex::VortexSessionDefault; @@ -258,10 +253,14 @@ mod tests { true, ), ])); - let layout = - ChunkedLayout::new(0, DType::from_arrow(schema.clone()), Arc::new(Empty)).into_layout(); let runtime = CurrentThreadRuntime::new(); let session = VortexSession::default().with_handle(runtime.handle()); + let layout = ChunkedLayout::new( + 0, + session.arrow().from_arrow_schema(&schema).unwrap(), + Arc::new(Empty), + ) + .into_layout(); let array = read_layout( &runtime, &session, diff --git a/crates/paimon/src/arrow/mod.rs b/crates/paimon/src/arrow/mod.rs index fddf9f193..e861c591c 100644 --- a/crates/paimon/src/arrow/mod.rs +++ b/crates/paimon/src/arrow/mod.rs @@ -125,7 +125,7 @@ pub fn paimon_type_to_arrow(dt: &PaimonDataType) -> crate::Result "entries", ArrowDataType::Struct( vec![ - ArrowField::new("key", element_type, m.element_type().is_nullable()), + ArrowField::new("key", element_type, false), ArrowField::new("value", ArrowDataType::Int32, false), ] .into(), @@ -435,7 +435,7 @@ pub(crate) fn paimon_type_to_arrow_for_read(dt: &PaimonDataType) -> crate::Resul ArrowField::new( "key", paimon_type_to_arrow_for_read(multiset.element_type())?, - multiset.element_type().is_nullable(), + false, ), ArrowField::new("value", ArrowDataType::Int32, false), ] @@ -556,6 +556,42 @@ mod tests { } } + #[test] + fn map_and_multiset_keys_are_non_nullable_in_read_and_write_schemas() { + for (data_type, value_nullable) in [ + ( + PaimonDataType::Map(MapType::new( + PaimonDataType::Int(IntType::new()), + PaimonDataType::Int(IntType::new()), + )), + true, + ), + ( + PaimonDataType::Multiset(crate::spec::MultisetType::new(PaimonDataType::Int( + IntType::new(), + ))), + false, + ), + ] { + let fields = vec![DataField::new(0, "values".into(), data_type)]; + for schema in [ + build_target_arrow_schema(&fields).unwrap(), + build_read_arrow_schema(&fields).unwrap(), + ] { + assert!(schema.field(0).is_nullable()); + let ArrowDataType::Map(entries, _) = schema.field(0).data_type() else { + panic!("MAP and MULTISET must both use Arrow Map storage"); + }; + assert!(!entries.is_nullable()); + let ArrowDataType::Struct(fields) = entries.data_type() else { + panic!("map entries must be a struct"); + }; + assert!(!fields[0].is_nullable()); + assert_eq!(fields[1].is_nullable(), value_nullable); + } + } + } + #[test] fn test_dictionary_resolves_to_value_type() { // A dictionary-encoded column carries the logical type of its values; diff --git a/crates/paimon/src/arrow/nested_evolution.rs b/crates/paimon/src/arrow/nested_evolution.rs index 2e511139e..015b2fee4 100644 --- a/crates/paimon/src/arrow/nested_evolution.rs +++ b/crates/paimon/src/arrow/nested_evolution.rs @@ -779,11 +779,7 @@ fn evolve_multiset( &map, elements, counts, - ArrowField::new( - "key", - paimon_type_to_arrow_for_read(target_element)?, - target_element.is_nullable(), - ), + ArrowField::new("key", paimon_type_to_arrow_for_read(target_element)?, false), ArrowField::new("value", ArrowDataType::Int32, false), ) } @@ -1911,7 +1907,7 @@ mod tests { Arc::new(StructArray::try_new(element_fields.clone(), vec![x], None).unwrap()); let counts: ArrayRef = Arc::new(Int32Array::from(vec![Some(1)])); let entry_fields = Fields::from(vec![ - ArrowField::new("key", ArrowDataType::Struct(element_fields.clone()), true), + ArrowField::new("key", ArrowDataType::Struct(element_fields.clone()), false), ArrowField::new("value", ArrowDataType::Int32, false), ]); let entries = diff --git a/crates/paimon/src/arrow/row_filter.rs b/crates/paimon/src/arrow/row_filter.rs index ff973e101..ace249888 100644 --- a/crates/paimon/src/arrow/row_filter.rs +++ b/crates/paimon/src/arrow/row_filter.rs @@ -50,4 +50,22 @@ pub trait RowFilter: Send + 'static { /// particular file. pub trait RowFilterFactory: Debug + Send + Sync { fn create(&self, context: RowFilterContext<'_>) -> crate::Result>>; + + /// Whether to refresh pruning predicates before reading each Parquet row group. + fn supports_row_group_pruning(&self) -> bool { + false + } + + /// A conservative snapshot of this filter, expressed against the historical + /// file fields. Called before fetching a row group's data, so changing TopK + /// thresholds can skip unread groups. Unsupported expressions must return + /// no predicates; errors fall back to reading the group. Exact row filtering + /// remains the caller's responsibility. Once a snapshot excludes a row, + /// later snapshots must not admit it again: skipped groups cannot be replayed. + fn pruning_predicates( + &self, + _file_fields: &[crate::spec::DataField], + ) -> crate::Result> { + Ok(Vec::new()) + } } diff --git a/crates/paimon/src/table/write_batch_normalize.rs b/crates/paimon/src/table/write_batch_normalize.rs index 36bd939c7..13337c613 100644 --- a/crates/paimon/src/table/write_batch_normalize.rs +++ b/crates/paimon/src/table/write_batch_normalize.rs @@ -1548,7 +1548,10 @@ mod tests { let output = normalize_write_array(&input, &expected).unwrap(); assert_eq!(output.data_type(), &expected); assert_eq!(output.len(), input.len()); - assert_eq!(output.nulls(), input.nulls()); + assert_eq!(output.null_count(), input.null_count()); + for row in 0..input.len() { + assert_eq!(output.is_null(row), input.is_null(row)); + } let data = output.to_data(); data.validate_full().unwrap(); } diff --git a/crates/paimon/testdata/vortex_compat/generate.py b/crates/paimon/testdata/vortex_compat/generate.py new file mode 100644 index 000000000..95eb75751 --- /dev/null +++ b/crates/paimon/testdata/vortex_compat/generate.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Generate with Python 3.11, vortex-data==0.75.0 and pyarrow==23.0.1.""" +from pathlib import Path +import pyarrow as pa +import vortex +import vortex.io + +assert vortex.__version__ == "0.75.0" +profile_type = pa.struct([pa.field("detail", pa.struct([pa.field("keep", pa.int32())]))]) +profiles = [{"detail": {"keep": 7}}, {"detail": None}, None, {"detail": {"keep": 9}}] * 128 +table = pa.table({ + "id": pa.array(range(512), type=pa.int32()), + "name": pa.array(["one", None, "three", "four"] * 128), + "profile": pa.array(profiles, type=profile_type), +}) +vortex.io.write(vortex.array(table), str(Path(__file__).with_name("vortex_0_75.vortex"))) diff --git a/crates/paimon/testdata/vortex_compat/vortex_0_75.vortex b/crates/paimon/testdata/vortex_compat/vortex_0_75.vortex new file mode 100644 index 000000000..e965ff667 Binary files /dev/null and b/crates/paimon/testdata/vortex_compat/vortex_0_75.vortex differ diff --git a/deny.toml b/deny.toml index 59bd5bdda..b0e6ce62b 100644 --- a/deny.toml +++ b/deny.toml @@ -18,14 +18,11 @@ [advisories] ignore = [ { id = "RUSTSEC-2023-0071", reason = "No fixed version; reqsign only signs locally generated outbound requests and exposes no remote signing or decryption timing oracle." }, - { id = "RUSTSEC-2026-0176", reason = "The affected PyList and PyTuple iterator APIs are unused." }, - { id = "RUSTSEC-2026-0177", reason = "PyCFunction::new_closure is unused." }, # zstd 0.13 is still required by Arrow, DataFusion, Parquet, and other upstream crates. # Revisit these API-specific exceptions as those dependencies adopt zstd 0.14. { id = "RUSTSEC-2026-0391", reason = "No caller uses the affected raw Encoder/Decoder::with_context constructors." }, { id = "RUSTSEC-2026-0398", reason = "The affected zstd-safe seekable feature is not enabled." }, { id = "RUSTSEC-2026-0403", reason = "No old zstd caller uses write::Decoder or raw streaming decode; compression-codecs now uses patched zstd-safe 8.2." }, - { id = "RUSTSEC-2026-0435", reason = "Parquet 58 pins thrift 0.17 and only uses the compact protocol; the affected TBinaryInputProtocol::read_bytes is unused." }, ] [licenses] diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 5bcce1a87..67f564a10 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -148,6 +148,14 @@ of at most a quarter of the limit and closes the row group once its encoded size reaches the limit, so large or skewed batches stay close to it. These options affect new writes only. +## Map and Multiset Arrow Values + +`MAP` and `MULTISET` both use Arrow Map with a non-null entry struct +and non-null keys. A multiset's values are non-null `Int32` occurrence counts. +Column and ordinary map value nullability follow the existing schema rules. +NULL keys are rejected by Arrow's runtime validation. +Row, Avro, and Parquet retain their Java-compatible file encodings. + ## Parquet Page Pruning Parquet Page Index pruning is enabled by default. Set diff --git a/docs/src/sql.md b/docs/src/sql.md index f3216c355..156f52895 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -27,7 +27,7 @@ under the License. [dependencies] paimon = "0.6.0" paimon-datafusion = "0.6.0" -datafusion = "54.0.0" +datafusion = "55.2.0" tokio = { version = "1", features = ["full"] } ``` @@ -36,7 +36,7 @@ and writes to ordinary Paimon tables can create them with `file.format=mosaic`. ## SQL Support Scope -`paimon-datafusion` currently targets Apache DataFusion 54.x. The workspace pins `datafusion = "54.0.0"`. +`paimon-datafusion` currently targets Apache DataFusion 55.x. The workspace uses `datafusion = "55.2.0"`. SQL support has two layers: @@ -47,6 +47,42 @@ Not every DataFusion DDL/DML statement maps to a Paimon table operation. For Pai For the exact delegated SQL grammar, see the [DataFusion SQL Reference](https://datafusion.apache.org/user-guide/sql/index.html). +### DataFusion 55 integration + +Paimon Parquet scans refresh supported dynamic predicates before reading each +remaining row group. This lets an `ORDER BY ... LIMIT` threshold skip groups +that can no longer contribute. Missing statistics, unsupported predicates, and +snapshot errors keep the group readable; the original runtime filter still +checks decoded rows. Floating-point predicates stay with DataFusion's row +filter because Parquet bounds do not account for NaN ordering. + +The NULL-propagating `parse_json`, `try_parse_json`, `variant_get`, +`try_variant_get`, `path_to_descriptor`, `descriptor_to_string`, and `blob_view` +functions declare DataFusion's strict UDF metadata. This enables outer join +elimination when a filter rejects their NULL results. `is_variant_null` keeps +its distinct SQL NULL behavior. + +Regular and audit-log scans report actual output rows, batches, bytes, and +compute time to DataFusion. For example, PostgreSQL-compatible JSON plans can +be requested with: + +```sql +EXPLAIN (ANALYZE, FORMAT pgjson, METRICS 'rows') +SELECT * FROM paimon.default.my_table WHERE id > 10; +``` + +DataFusion's built-in array functions (including `array_scale`, `array_sum`, +and `unnest_outer`) and its planner, aggregation, and spill improvements are +available through the normal session. Custom spill backends can be configured +on the `RuntimeEnv` passed to `SQLContext::builder`. + +Paimon `MERGE INTO` continues to use `SQLContext`'s Paimon write implementation. +DataFusion's generic `TableProvider::merge_into` hook, `file_row_index()` and +`input_file_name()` for Paimon scans, range-partition declarations, and custom +query-planner FFI are not provided by this upgrade. In particular, Paimon's +logical row IDs are not physical file row offsets, and ordinary scan splits +do not guarantee disjoint sorted ranges. + ## Registering Catalog Register an entire Paimon catalog so all databases and tables are accessible via `paimon.database.table` syntax: