diff --git a/Cargo.lock b/Cargo.lock index a11a6fa..791e333 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,6 +41,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "0.6.21" @@ -108,7 +114,7 @@ dependencies = [ "hex", "lzma-rust2", "rayon", - "sha2", + "sha2 0.10.9", "shellexpand", "tempfile", "ureq", @@ -129,7 +135,7 @@ dependencies = [ "lazy_static", "opendal", "serde", - "sha2", + "sha2 0.10.9", "tokio", "tokio-stream", "tokio-util", @@ -265,6 +271,18 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "benches" +version = "0.1.0" +dependencies = [ + "criterion", + "opendal", + "rand 0.9.2", + "sha2 0.11.0", + "tempdir", + "tokio", +] + [[package]] name = "bitflags" version = "2.10.0" @@ -280,6 +298,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + [[package]] name = "brotli" version = "8.0.2" @@ -319,6 +346,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.45" @@ -356,6 +389,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.5.51" @@ -415,7 +475,7 @@ dependencies = [ "lzma-rust2", "opendal", "serde", - "sha2", + "sha2 0.10.9", "tokio", "zstd", ] @@ -446,6 +506,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -481,6 +547,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.3.0" @@ -514,6 +589,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -555,18 +666,38 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +dependencies = [ + "hybrid-array", +] + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.6", "subtle", ] +[[package]] +name = "digest" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4850db49bf08e663084f7fb5c87d202ef91a3907271aff24a94eb97ff039153c" +dependencies = [ + "block-buffer 0.12.0", + "const-oid 0.10.2", + "crypto-common 0.2.1", +] + [[package]] name = "dirs" version = "6.0.0" @@ -682,6 +813,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "futures" version = "0.3.31" @@ -820,6 +957,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -838,6 +986,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "hex" version = "0.4.3" @@ -850,7 +1004,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -908,6 +1062,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d46837a0ed51fe95bd3b05de33cd64a1ee88fc797477ca48446872504507c5" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.7.0" @@ -1129,12 +1292,32 @@ dependencies = [ "serde", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -1231,7 +1414,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bd57dc1513725e598b9609a9505d6fe28ee280861e5b4d792b3db9d00b1a315" dependencies = [ "crc", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -1256,7 +1439,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", ] [[package]] @@ -1322,6 +1505,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "opendal" version = "0.54.1" @@ -1436,6 +1625,34 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "potential_utf" version = "0.1.4" @@ -1566,6 +1783,19 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + [[package]] name = "rand" version = "0.8.5" @@ -1607,6 +1837,21 @@ dependencies = [ "rand_core 0.9.3", ] +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.6.4" @@ -1645,6 +1890,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1665,6 +1919,18 @@ dependencies = [ "thiserror", ] +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.14" @@ -1682,6 +1948,15 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "reqsign" version = "0.16.5" @@ -1707,7 +1982,7 @@ dependencies = [ "serde", "serde_json", "sha1", - "sha2", + "sha2 0.10.9", "tokio", ] @@ -1852,6 +2127,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1946,8 +2230,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] @@ -1957,11 +2241,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", "sha2-asm", ] +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.2", +] + [[package]] name = "sha2-asm" version = "0.6.4" @@ -2081,6 +2376,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + [[package]] name = "tempfile" version = "3.25.0" @@ -2142,6 +2447,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.10.0" @@ -2395,9 +2710,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" [[package]] name = "uncased" @@ -2503,6 +2818,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -2627,6 +2952,37 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" diff --git a/Cargo.toml b/Cargo.toml index 6958805..1ac8c82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["client", "common", "server"] +members = ["client", "common", "server", "benches"] [profile.release] lto = "thin" diff --git a/benches/Cargo.toml b/benches/Cargo.toml new file mode 100644 index 0000000..4e385ee --- /dev/null +++ b/benches/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "benches" +version = "0.1.0" +edition = "2021" + +[dependencies] +sha2 = "0.11.0" +criterion = { version = "0.5", features = ["html_reports"] } +rand = "0.9" +opendal = { version = "0.54.1", features = ["services-fs"] } +tempdir = "0.3.7" +tokio = { version = "1.45.1", features = ["full"] } + +#[target.'cfg(unix)'.dependencies] +#sha2 = { version = "0.10.9", features = ["asm"] } + +[[bench]] +name = "benchmarks" +harness = false + diff --git a/benches/benches/benchmarks.rs b/benches/benches/benchmarks.rs new file mode 100644 index 0000000..b0482a4 --- /dev/null +++ b/benches/benches/benchmarks.rs @@ -0,0 +1,276 @@ +use std::fs::{self, File}; +use std::hint::black_box; +use std::io::{Read, Write}; +use std::path::Path; +use std::time::Duration; + +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use opendal::{services, Operator}; +use rand::Rng; +use sha2::{Digest, Sha256, Sha512}; +use tempdir::TempDir; +use tokio::runtime::Runtime; + +const KIB: usize = 1024; +const MIB: usize = 1024 * KIB; +const GIB: usize = 1024 * MIB; +const TEN_GIB: usize = 10 * GIB; + +const CHUNK_SIZE: usize = 64 * KIB; +const MAGIC_HEADER: &[u8] = b"\xDEADBEEF"; + +struct FileAccessFixture { + root: TempDir, + file_names: Vec, + total_bytes: usize, + operator: Operator, + runtime: Runtime, +} + +impl FileAccessFixture { + fn path(&self) -> &Path { + self.root.path() + } +} + +fn build_random_data(size: usize) -> Vec { + let mut rng = rand::rng(); + (0..size).map(|_| rng.random::()).collect() +} + +fn write_file_with_repeated_chunk(path: &Path, size: usize, chunk: &[u8]) { + let mut file = File::create(path).expect("create fixture file"); + let mut remaining = size; + + while remaining > 0 { + let write_len = remaining.min(chunk.len()); + file.write_all(&chunk[..write_len]).expect("write fixture file"); + remaining -= write_len; + } + + file.sync_all().expect("sync fixture file"); +} + +fn build_file_access_fixture() -> FileAccessFixture { + let total_bytes: usize = 3 * GIB; + let file_size = 64 * MIB; + let file_count = total_bytes.div_ceil(file_size); + + let root = TempDir::new("file-access-bench").expect("create temp directory"); + + let generation_chunk = build_random_data(file_size); + + let mut file_names = Vec::with_capacity(file_count); + let mut remaining = total_bytes; + + for index in 0..file_count { + let file_bytes = remaining.min(file_size); + let file_name = format!("file-{index:04}.bin"); + + write_file_with_repeated_chunk(&root.path().join(&file_name), file_bytes, &generation_chunk); + + file_names.push(file_name); + remaining -= file_bytes; + } + + let runtime = Runtime::new().expect("create tokio runtime"); + let operator = Operator::new( + services::Fs::default().root(root.path().to_str().expect("temp path is valid utf-8")), + ) + .expect("create fs operator") + .finish(); + + FileAccessFixture { + root, + file_names, + total_bytes, + operator, + runtime, + } +} + +fn sha2_benchmarks(c: &mut Criterion) { + let chunk: Vec = { + let mut rng = rand::rng(); + (0..CHUNK_SIZE).map(|_| rng.random::()).collect() + }; + let iterations = TEN_GIB / CHUNK_SIZE; + + let mut group = c.benchmark_group("hash-10GiB"); + group.throughput(Throughput::Bytes(TEN_GIB as u64)); + group.sample_size(10); + group.measurement_time(Duration::from_secs(200)); + + group.bench_function(BenchmarkId::new("SHA-256", "10GiB"), |b| { + b.iter(|| { + let mut hasher = Sha256::new(); + for _ in 0..iterations { + hasher.update(&chunk); + } + hasher.finalize() + }); + }); + + group.bench_function(BenchmarkId::new("SHA-512", "10GiB"), |b| { + b.iter(|| { + let mut hasher = Sha512::new(); + for _ in 0..iterations { + hasher.update(&chunk); + } + hasher.finalize() + }); + }); + + group.finish(); +} + +fn file_access_benchmarks(c: &mut Criterion) { + let fixture = build_file_access_fixture(); + let file_size = fixture.total_bytes / fixture.file_names.len(); + + // --- Read --- + let mut group = c.benchmark_group("file-access-read"); + group.throughput(Throughput::Bytes(fixture.total_bytes as u64)); + group.sample_size(10); + group.measurement_time(Duration::from_secs(60)); + + group.bench_function(BenchmarkId::new("std", fixture.total_bytes), |b| { + b.iter(|| { + let mut total = 0usize; + for name in &fixture.file_names { + let mut buf = Vec::new(); + File::open(fixture.path().join(name)) + .expect("open file") + .read_to_end(&mut buf) + .expect("read file"); + total += buf.len(); + } + black_box(total) + }); + }); + + group.bench_function(BenchmarkId::new("opendal-fs", fixture.total_bytes), |b| { + b.iter(|| { + let total = fixture.runtime.block_on(async { + let mut total = 0usize; + for name in &fixture.file_names { + total += fixture.operator.read(name).await.expect("read file").len(); + } + total + }); + black_box(total) + }); + }); + group.finish(); + + // --- Exists --- + let mut group = c.benchmark_group("file-access-exists"); + group.throughput(Throughput::Elements(fixture.file_names.len() as u64)); + group.sample_size(10); + group.measurement_time(Duration::from_secs(20)); + + group.bench_function(BenchmarkId::new("std", fixture.file_names.len()), |b| { + b.iter(|| { + let count = fixture.file_names.iter() + .filter(|name| fixture.path().join(name).exists()) + .count(); + black_box(count) + }); + }); + + group.bench_function(BenchmarkId::new("opendal-fs", fixture.file_names.len()), |b| { + b.iter(|| { + let count = fixture.runtime.block_on(async { + let mut count = 0usize; + for name in &fixture.file_names { + if fixture.operator.exists(name).await.expect("stat file") { + count += 1; + } + } + count + }); + black_box(count) + }); + }); + group.finish(); + + // --- Write --- + let mut group = c.benchmark_group("file-access-write"); + group.throughput(Throughput::Bytes(fixture.total_bytes as u64)); + group.sample_size(10); + group.measurement_time(Duration::from_secs(60)); + + let write_chunk = build_random_data(file_size); + + group.bench_function(BenchmarkId::new("std", fixture.total_bytes), |b| { + b.iter(|| { + for name in &fixture.file_names { + let mut file = File::create(fixture.path().join(name)).expect("create file"); + file.write_all(&write_chunk).expect("write file"); + file.sync_all().expect("sync file"); + } + black_box(fixture.total_bytes) + }); + }); + + group.bench_function(BenchmarkId::new("opendal-fs", fixture.total_bytes), |b| { + b.iter(|| { + fixture.runtime.block_on(async { + for name in &fixture.file_names { + fixture + .operator + .write(name, write_chunk.clone()) + .await + .expect("write file"); + } + }); + black_box(fixture.total_bytes) + }); + }); + group.finish(); + + // --- Copy --- + let copy_dest = fixture.path().join("copy-dest"); + + let mut group = c.benchmark_group("file-access-copy"); + group.throughput(Throughput::Bytes(fixture.total_bytes as u64)); + group.sample_size(10); + group.measurement_time(Duration::from_secs(60)); + + group.bench_function(BenchmarkId::new("std", fixture.total_bytes), |b| { + b.iter(|| { + fs::create_dir_all(©_dest).expect("create copy dir"); + let mut copied = 0usize; + for name in &fixture.file_names { + let data = fs::read(fixture.path().join(name)).expect("read file"); + let mut out = File::create(copy_dest.join(name)).expect("create file"); + out.write_all(MAGIC_HEADER).expect("write header"); + out.write_all(&data).expect("write data"); + copied += data.len(); + } + fs::remove_dir_all(©_dest).expect("remove copy dir"); + black_box(copied) + }); + }); + + group.bench_function(BenchmarkId::new("opendal-fs", fixture.total_bytes), |b| { + b.iter(|| { + fixture.runtime.block_on(async { + for name in &fixture.file_names { + let data = fixture.operator.read(name).await.expect("read file"); + let dest = format!("copy-dest/{name}"); + let mut out = Vec::with_capacity(MAGIC_HEADER.len() + data.len()); + out.extend_from_slice(MAGIC_HEADER); + out.extend_from_slice(&data.to_vec()); + fixture.operator.write(&dest, out).await.expect("write file"); + } + }); + let _ = fs::remove_dir_all(©_dest); + black_box(fixture.total_bytes) + }); + }); + group.finish(); +} + +criterion_group!(benches, sha2_benchmarks, file_access_benchmarks); +criterion_main!(benches); \ No newline at end of file