From 08bd6ed31d122271412cd703a264263fba962f3d Mon Sep 17 00:00:00 2001 From: Hamish Peebles Date: Sat, 25 Jul 2026 11:37:02 +0100 Subject: [PATCH 1/5] perf(btreemap): add `insert_many` for batched insertion `insert` writes the target node to stable memory on every call, and when a node splits it writes the new sibling and the parent too. Inserting a run of keys therefore rewrites the same nodes repeatedly: building a 10k-entry map one key at a time performs 15,976 node writes across just 1,997 distinct nodes. `insert_many` keeps the current root-to-leaf path in memory and writes a node out only once the input has moved past it, bringing those same 10k keys down to 4,032 writes. The header is likewise written once per call rather than once per key. map.insert_many((0..1_000_000).map(|i| (i, compute(i)))); The iterator is consumed lazily and never collected, so the memory cost is one tree path however many pairs are supplied. ## Ordering Coalescing pays off when consecutive keys land in the same node, so keys should be supplied in sorted order. Direction is irrelevant: a descending run unwinds the path leftwards instead of rightwards and produces an identical write count. Out-of-order keys stay correct rather than corrupting the tree. Each level of the held path records the key range its node owns at both ends, so a key arriving early unwinds to an ancestor that covers it, as far as the root if necessary. Tracking the lower bound as well as the upper one costs a comparison per key. ## Measurements canbench, 10k u64 pairs each: ascending, empty map 334.19M -> 76.91M instructions (4.3x) descending, empty map 386.65M -> 96.43M instructions (4.0x) unordered, caller sorts 356.43M -> 86.45M instructions (4.1x) unordered, fed as-is 356.43M -> 341.07M instructions (1.0x) ascending run into 100k 294.07M -> 49.06M instructions (6.0x) The sort in the third row is inside the measured section. Unordered input fed straight in forfeits the coalescing, but is not slower than the insert loop it replaces. ## Implementation notes Holding the whole path rather than only the leaf is what makes this work. Two thirds of the writes in a split-heavy workload are the sibling and the parent, and a held parent absorbs many child splits before it is written; holding only the leaf coalesces 2x rather than 4x. Node splits are handled in place while the path is held, except in two cases that fall back to the ordinary insert path: a full root leaf, and a parent with no room for a promoted median. Those fire 277 times per 10k ascending keys, about one insert in 36. A fallback flushes the path, so nodes it had buffered are written again when the batch re-descends into them; that accounts for the whole gap between the 4,032 writes above and the 1,997 that writing each dirty node exactly once would take. Handling the split cascade in place would close it, at the cost of considerably more intricate code in the riskiest part of the tree, so it is left as a possible follow-up. The batch machinery lives in `src/btreemap/bulk_insert.rs`, leaving `BTreeMap::insert_many` in `btreemap.rs` as a thin wrapper over it. Dropping `BulkInsert` writes back whatever it still holds, which is how a batch is completed: the drop is load-bearing on every call. `insert` is unchanged; its body moves into `insert_serialized` so the fallback can reuse it with an already-serialized value. All 252 pre-existing benchmarks are unchanged to within noise. ## Testing Differential proptests and unit tests build the same map with repeated `insert` and with `insert_many`, and require the results to be indistinguishable: across node cache sizes 0/1/16, the V1, V1-migrated-to-V2 and V2 layouts, unbounded values large enough to spill onto overflow pages, duplicate and overwriting batches, and batch sizes chosen to hit both fallbacks. Ordering is covered by ascending, descending, zigzag, sawtooth and shuffled inputs. Allocator chunk counts return to zero after draining a map built by `insert_many`. Co-Authored-By: Claude Opus 5 --- benchmarks/btreemap/canbench_results.yml | 356 +++++++++++++---------- benchmarks/btreemap/src/main.rs | 108 +++++++ src/btreemap.rs | 71 ++++- src/btreemap/bulk_insert.rs | 310 ++++++++++++++++++++ src/btreemap/proptests.rs | 84 ++++++ src/btreemap/tests.rs | 249 ++++++++++++++++ 6 files changed, 1022 insertions(+), 156 deletions(-) create mode 100644 src/btreemap/bulk_insert.rs diff --git a/benchmarks/btreemap/canbench_results.yml b/benchmarks/btreemap/canbench_results.yml index 108c18e9..eb0f84f6 100644 --- a/benchmarks/btreemap/canbench_results.yml +++ b/benchmarks/btreemap/canbench_results.yml @@ -51,21 +51,21 @@ benches: btreemap_v2_contains_then_remove_blob_32_128: total: calls: 1 - instructions: 735347850 + instructions: 735347859 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_contains_then_remove_u64_u64: total: calls: 1 - instructions: 597959237 + instructions: 598024460 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_contains_then_remove_u64_u64_nocache: total: calls: 1 - instructions: 788784522 + instructions: 788849745 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -184,7 +184,7 @@ benches: btreemap_v2_get_blob8_u64: total: calls: 1 - instructions: 205616407 + instructions: 206168780 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -289,7 +289,7 @@ benches: btreemap_v2_get_blob_4_128: total: calls: 1 - instructions: 169181540 + instructions: 169374118 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -345,42 +345,42 @@ benches: btreemap_v2_get_then_insert_blob_32_128: total: calls: 1 - instructions: 525765859 + instructions: 525735862 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_get_then_insert_u64_u64: total: calls: 1 - instructions: 375030642 + instructions: 375267793 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_get_then_insert_u64_u64_nocache: total: calls: 1 - instructions: 604576977 + instructions: 604814288 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_get_then_remove_blob_32_128: total: calls: 1 - instructions: 750750883 + instructions: 750750892 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_get_then_remove_u64_u64: total: calls: 1 - instructions: 602628511 + instructions: 602693734 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_get_then_remove_u64_u64_nocache: total: calls: 1 - instructions: 793317078 + instructions: 793382301 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -408,7 +408,7 @@ benches: btreemap_v2_get_vec8_u64: total: calls: 1 - instructions: 265721428 + instructions: 268419835 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -576,343 +576,399 @@ benches: btreemap_v2_insert_100k_u64_u64: total: calls: 1 - instructions: 4291778109 + instructions: 4294470481 heap_increase: 0 stable_memory_increase: 60 scopes: {} btreemap_v2_insert_10mib_values: total: calls: 1 - instructions: 4375302957 + instructions: 4370305256 heap_increase: 161 stable_memory_increase: 3613 scopes: {} + btreemap_v2_insert_batch_into_existing_u64_u64: + total: + calls: 1 + instructions: 294066274 + heap_increase: 0 + stable_memory_increase: 0 + scopes: {} + btreemap_v2_insert_batch_random_u64_u64: + total: + calls: 1 + instructions: 356430975 + heap_increase: 0 + stable_memory_increase: 6 + scopes: {} btreemap_v2_insert_blob8_u64: total: calls: 1 - instructions: 342831527 + instructions: 343422096 heap_increase: 0 stable_memory_increase: 4 scopes: {} btreemap_v2_insert_blob_1024_128: total: calls: 1 - instructions: 4089419201 + instructions: 4089409188 heap_increase: 2 stable_memory_increase: 196 scopes: {} btreemap_v2_insert_blob_128_128: total: calls: 1 - instructions: 898938251 + instructions: 898928072 heap_increase: 0 stable_memory_increase: 46 scopes: {} btreemap_v2_insert_blob_16_128: total: calls: 1 - instructions: 397233634 + instructions: 397221173 heap_increase: 0 stable_memory_increase: 24 scopes: {} btreemap_v2_insert_blob_256_128: total: calls: 1 - instructions: 1385784168 + instructions: 1385774087 heap_increase: 0 stable_memory_increase: 67 scopes: {} btreemap_v2_insert_blob_32_0: total: calls: 1 - instructions: 404010801 + instructions: 403980756 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_blob_32_1024: total: calls: 1 - instructions: 640535326 + instructions: 640524429 heap_increase: 0 stable_memory_increase: 173 scopes: {} btreemap_v2_insert_blob_32_128: total: calls: 1 - instructions: 448746352 + instructions: 448735453 heap_increase: 0 stable_memory_increase: 28 scopes: {} btreemap_v2_insert_blob_32_16: total: calls: 1 - instructions: 438405771 + instructions: 438414948 heap_increase: 0 stable_memory_increase: 11 scopes: {} btreemap_v2_insert_blob_32_256: total: calls: 1 - instructions: 479120078 + instructions: 479109269 heap_increase: 0 stable_memory_increase: 49 scopes: {} btreemap_v2_insert_blob_32_32: total: calls: 1 - instructions: 433815169 + instructions: 433824242 heap_increase: 0 stable_memory_increase: 13 scopes: {} btreemap_v2_insert_blob_32_4: total: calls: 1 - instructions: 420418817 + instructions: 420408336 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_blob_32_512: total: calls: 1 - instructions: 543376008 + instructions: 543365153 heap_increase: 0 stable_memory_increase: 91 scopes: {} btreemap_v2_insert_blob_32_64: total: calls: 1 - instructions: 447280905 + instructions: 447289930 heap_increase: 0 stable_memory_increase: 18 scopes: {} btreemap_v2_insert_blob_32_8: total: calls: 1 - instructions: 423948483 + instructions: 423937590 heap_increase: 0 stable_memory_increase: 9 scopes: {} btreemap_v2_insert_blob_4_128: total: calls: 1 - instructions: 325528558 + instructions: 325959134 heap_increase: 0 stable_memory_increase: 13 scopes: {} btreemap_v2_insert_blob_512_128: total: calls: 1 - instructions: 2240845938 + instructions: 2240835899 heap_increase: 1 stable_memory_increase: 111 scopes: {} btreemap_v2_insert_blob_64_128: total: calls: 1 - instructions: 565741268 + instructions: 565730841 heap_increase: 0 stable_memory_increase: 34 scopes: {} btreemap_v2_insert_blob_8_128: total: calls: 1 - instructions: 370623171 + instructions: 370608603 heap_increase: 0 stable_memory_increase: 20 scopes: {} + btreemap_v2_insert_desc_u64_u64: + total: + calls: 1 + instructions: 386646941 + heap_increase: 0 + stable_memory_increase: 8 + scopes: {} + btreemap_v2_insert_many_desc_u64_u64: + total: + calls: 1 + instructions: 96826813 + heap_increase: 0 + stable_memory_increase: 8 + scopes: {} + btreemap_v2_insert_many_into_existing_u64_u64: + total: + calls: 1 + instructions: 49506647 + heap_increase: 0 + stable_memory_increase: 0 + scopes: {} + btreemap_v2_insert_many_seq_u64_u64: + total: + calls: 1 + instructions: 77307535 + heap_increase: 0 + stable_memory_increase: 8 + scopes: {} + btreemap_v2_insert_many_sorted_random_u64_u64: + total: + calls: 1 + instructions: 86845730 + heap_increase: 3 + stable_memory_increase: 8 + scopes: {} + btreemap_v2_insert_many_unsorted_u64_u64: + total: + calls: 1 + instructions: 343377265 + heap_increase: 0 + stable_memory_increase: 6 + scopes: {} btreemap_v2_insert_overwrite_u64_u64: total: calls: 1 - instructions: 296057608 + instructions: 296294757 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_insert_overwrite_u64_u64_nocache: total: calls: 1 - instructions: 371438961 + instructions: 371676272 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_insert_overwrite_zipf_10k_u64_u64: total: calls: 1 - instructions: 265045208 + instructions: 265316225 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_insert_principal: total: calls: 1 - instructions: 397790991 + instructions: 397834639 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_seq_u64_u64: total: calls: 1 - instructions: 333906568 + instructions: 334191577 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_seq_u64_u64_nocache: total: calls: 1 - instructions: 472716458 + instructions: 473001467 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_u64_blob8: total: calls: 1 - instructions: 352838177 + instructions: 352851122 heap_increase: 0 stable_memory_increase: 5 scopes: {} btreemap_v2_insert_u64_u64: total: calls: 1 - instructions: 356162259 + instructions: 356430975 heap_increase: 0 stable_memory_increase: 6 scopes: {} btreemap_v2_insert_u64_vec8: total: calls: 1 - instructions: 351056169 + instructions: 351036169 heap_increase: 0 stable_memory_increase: 21 scopes: {} btreemap_v2_insert_vec8_u64: total: calls: 1 - instructions: 434940532 + instructions: 434503480 heap_increase: 0 stable_memory_increase: 16 scopes: {} btreemap_v2_insert_vec_1024_128: total: calls: 1 - instructions: 2192230335 + instructions: 2194429661 heap_increase: 0 stable_memory_increase: 193 scopes: {} btreemap_v2_insert_vec_128_128: total: calls: 1 - instructions: 862859491 + instructions: 864135973 heap_increase: 0 stable_memory_increase: 51 scopes: {} btreemap_v2_insert_vec_16_128: total: calls: 1 - instructions: 563612378 + instructions: 563873345 heap_increase: 0 stable_memory_increase: 31 scopes: {} btreemap_v2_insert_vec_256_128: total: calls: 1 - instructions: 1181410426 + instructions: 1183331041 heap_increase: 0 stable_memory_increase: 71 scopes: {} btreemap_v2_insert_vec_32_0: total: calls: 1 - instructions: 510924354 + instructions: 510952957 heap_increase: 0 stable_memory_increase: 20 scopes: {} btreemap_v2_insert_vec_32_1024: total: calls: 1 - instructions: 1049358367 + instructions: 1050297481 heap_increase: 0 stable_memory_increase: 171 scopes: {} btreemap_v2_insert_vec_32_128: total: calls: 1 - instructions: 634848625 + instructions: 635239590 heap_increase: 0 stable_memory_increase: 33 scopes: {} btreemap_v2_insert_vec_32_16: total: calls: 1 - instructions: 562776441 + instructions: 562766441 heap_increase: 0 stable_memory_increase: 20 scopes: {} btreemap_v2_insert_vec_32_256: total: calls: 1 - instructions: 738460103 + instructions: 739231709 heap_increase: 0 stable_memory_increase: 54 scopes: {} btreemap_v2_insert_vec_32_32: total: calls: 1 - instructions: 556986479 + instructions: 556976479 heap_increase: 0 stable_memory_increase: 20 scopes: {} btreemap_v2_insert_vec_32_4: total: calls: 1 - instructions: 539343255 + instructions: 539333255 heap_increase: 0 stable_memory_increase: 20 scopes: {} btreemap_v2_insert_vec_32_512: total: calls: 1 - instructions: 834725813 + instructions: 835651222 heap_increase: 0 stable_memory_increase: 91 scopes: {} btreemap_v2_insert_vec_32_64: total: calls: 1 - instructions: 564203551 + instructions: 564249167 heap_increase: 0 stable_memory_increase: 24 scopes: {} btreemap_v2_insert_vec_32_8: total: calls: 1 - instructions: 533050455 + instructions: 533040455 heap_increase: 0 stable_memory_increase: 20 scopes: {} btreemap_v2_insert_vec_4_128: total: calls: 1 - instructions: 436367493 + instructions: 436519587 heap_increase: 1 stable_memory_increase: 16 scopes: {} btreemap_v2_insert_vec_512_128: total: calls: 1 - instructions: 1567958241 + instructions: 1570135282 heap_increase: 0 stable_memory_increase: 112 scopes: {} btreemap_v2_insert_vec_64_128: total: calls: 1 - instructions: 722228540 + instructions: 722888819 heap_increase: 0 stable_memory_increase: 41 scopes: {} btreemap_v2_insert_vec_8_128: total: calls: 1 - instructions: 528020057 + instructions: 528253615 heap_increase: 0 stable_memory_increase: 23 scopes: {} @@ -996,126 +1052,126 @@ benches: btreemap_v2_mem_manager_get_u64_u64: total: calls: 1 - instructions: 213714676 + instructions: 213677620 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mem_manager_get_u64_vec512: total: calls: 1 - instructions: 277358190 + instructions: 276522702 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mem_manager_get_vec512_u64: total: calls: 1 - instructions: 938767815 + instructions: 937902467 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mem_manager_insert_blob512_u64: total: calls: 1 - instructions: 2319454087 + instructions: 2319424809 heap_increase: 1 stable_memory_increase: 0 scopes: {} btreemap_v2_mem_manager_insert_u64_blob512: total: calls: 1 - instructions: 534153431 + instructions: 534143599 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mem_manager_insert_u64_u64: total: calls: 1 - instructions: 434571754 + instructions: 434555466 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mem_manager_insert_u64_vec512: total: calls: 1 - instructions: 736566555 + instructions: 736567925 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mem_manager_insert_vec512_u64: total: calls: 1 - instructions: 1618293038 + instructions: 1619612768 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mem_manager_remove_blob512_u64: total: calls: 1 - instructions: 3524433860 + instructions: 3524412823 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mem_manager_remove_u64_blob512: total: calls: 1 - instructions: 822448050 + instructions: 822426155 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mem_manager_remove_u64_u64: total: calls: 1 - instructions: 660790252 + instructions: 660806415 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mem_manager_remove_u64_vec512: total: calls: 1 - instructions: 1148510151 + instructions: 1148784099 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mem_manager_remove_vec512_u64: total: calls: 1 - instructions: 2753136803 + instructions: 2755712871 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mixed_get_insert_zipf_10k_u64_u64: total: calls: 1 - instructions: 337367150 + instructions: 337669142 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mixed_get_insert_zipf_10k_u64_u64_nocache: total: calls: 1 - instructions: 576031481 + instructions: 576288777 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_peek_then_pop_first_u64_u64: total: calls: 1 - instructions: 451846877 + instructions: 451906395 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_peek_then_pop_last_u64_u64: total: calls: 1 - instructions: 445250519 + instructions: 445312047 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_peek_then_pop_last_u64_u64_nocache: total: calls: 1 - instructions: 701932179 + instructions: 701993707 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1129,7 +1185,7 @@ benches: btreemap_v2_pop_first_blob_32_0: total: calls: 1 - instructions: 398686459 + instructions: 398689253 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1150,35 +1206,35 @@ benches: btreemap_v2_pop_first_blob_8_128: total: calls: 1 - instructions: 337064290 + instructions: 337064326 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_pop_first_principal: total: calls: 1 - instructions: 414634432 + instructions: 414669002 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_pop_first_u64_u64: total: calls: 1 - instructions: 404184794 + instructions: 404244312 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_pop_first_vec_32_128: total: calls: 1 - instructions: 677735066 + instructions: 678205549 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_pop_first_vec_32_vec128: total: calls: 1 - instructions: 677735066 + instructions: 678205549 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1192,14 +1248,14 @@ benches: btreemap_v2_pop_last_blob_32_0: total: calls: 1 - instructions: 366514459 + instructions: 366517061 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_pop_last_blob_32_1024: total: calls: 1 - instructions: 714422032 + instructions: 714422041 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1213,133 +1269,133 @@ benches: btreemap_v2_pop_last_blob_8_128: total: calls: 1 - instructions: 319261539 + instructions: 319261564 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_pop_last_principal: total: calls: 1 - instructions: 387499419 + instructions: 387532456 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_pop_last_u64_u64: total: calls: 1 - instructions: 390486139 + instructions: 390547667 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_pop_last_vec_32_128: total: calls: 1 - instructions: 656405996 + instructions: 656879323 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_pop_last_vec_32_vec128: total: calls: 1 - instructions: 656405996 + instructions: 656879323 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_range_count_1k_0b: total: calls: 1 - instructions: 17438 + instructions: 17415 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_range_count_1k_10kib: total: calls: 1 - instructions: 2403569 + instructions: 2410332 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_range_count_20_10mib: total: calls: 1 - instructions: 18468765 + instructions: 18468866 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_range_key_sum_1k_0b: total: calls: 1 - instructions: 17409 + instructions: 17392 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_range_key_sum_1k_10kib: total: calls: 1 - instructions: 2469367 + instructions: 2476135 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_range_key_sum_20_10mib: total: calls: 1 - instructions: 18469999 + instructions: 18470105 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_range_small_u64_u64: total: calls: 1 - instructions: 18575496 + instructions: 18552447 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_range_value_sum_1k_0b: total: calls: 1 - instructions: 17776 + instructions: 17759 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_range_value_sum_1k_10kib: total: calls: 1 - instructions: 20796260 + instructions: 20803028 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_range_value_sum_20_10mib: total: calls: 1 - instructions: 398305226 + instructions: 398305332 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_100k_u64_u64: total: calls: 1 - instructions: 6459956144 + instructions: 6460531255 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_100k_u64_u64_nocache: total: calls: 1 - instructions: 7007814665 + instructions: 7008389776 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_10mib_values: total: calls: 1 - instructions: 4705256210 + instructions: 4699761840 heap_increase: 0 stable_memory_increase: 657 scopes: {} btreemap_v2_remove_blob8_u64: total: calls: 1 - instructions: 494480126 + instructions: 494421991 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1374,21 +1430,21 @@ benches: btreemap_v2_remove_blob_32_0: total: calls: 1 - instructions: 590771254 + instructions: 590772401 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_blob_32_1024: total: calls: 1 - instructions: 941187011 + instructions: 941187020 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_blob_32_128: total: calls: 1 - instructions: 655869991 + instructions: 655870000 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1402,7 +1458,7 @@ benches: btreemap_v2_remove_blob_32_256: total: calls: 1 - instructions: 700056800 + instructions: 700056855 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1416,14 +1472,14 @@ benches: btreemap_v2_remove_blob_32_4: total: calls: 1 - instructions: 617487671 + instructions: 617488366 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_blob_32_512: total: calls: 1 - instructions: 800021230 + instructions: 800021239 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1437,14 +1493,14 @@ benches: btreemap_v2_remove_blob_32_8: total: calls: 1 - instructions: 614281188 + instructions: 614281321 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_blob_4_128: total: calls: 1 - instructions: 373323149 + instructions: 373429303 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1472,21 +1528,21 @@ benches: btreemap_v2_remove_principal: total: calls: 1 - instructions: 598900093 + instructions: 598932128 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_u64_blob8: total: calls: 1 - instructions: 524277916 + instructions: 524286026 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_u64_u64: total: calls: 1 - instructions: 533948350 + instructions: 534013573 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1500,35 +1556,35 @@ benches: btreemap_v2_remove_vec8_u64: total: calls: 1 - instructions: 638942966 + instructions: 639504895 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_vec_1024_128: total: calls: 1 - instructions: 3955976774 + instructions: 3959893383 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_vec_128_128: total: calls: 1 - instructions: 1290393122 + instructions: 1292449064 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_vec_16_128: total: calls: 1 - instructions: 792212701 + instructions: 792538911 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_vec_256_128: total: calls: 1 - instructions: 1974882757 + instructions: 1978175955 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1542,14 +1598,14 @@ benches: btreemap_v2_remove_vec_32_1024: total: calls: 1 - instructions: 1545910320 + instructions: 1547504634 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_vec_32_128: total: calls: 1 - instructions: 902750586 + instructions: 903286470 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1563,7 +1619,7 @@ benches: btreemap_v2_remove_vec_32_256: total: calls: 1 - instructions: 1109185713 + instructions: 1110457553 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1584,14 +1640,14 @@ benches: btreemap_v2_remove_vec_32_512: total: calls: 1 - instructions: 1267880569 + instructions: 1269457668 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_vec_32_64: total: calls: 1 - instructions: 840262373 + instructions: 840343517 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1605,161 +1661,161 @@ benches: btreemap_v2_remove_vec_4_128: total: calls: 1 - instructions: 511552975 + instructions: 511685524 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_vec_512_128: total: calls: 1 - instructions: 2719139728 + instructions: 2722977588 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_vec_64_128: total: calls: 1 - instructions: 1040869622 + instructions: 1041831057 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_vec_8_128: total: calls: 1 - instructions: 700913260 + instructions: 701159221 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_remove_zipf_10k_u64_u64: total: calls: 1 - instructions: 272321922 + instructions: 272334672 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_iter_1k_0b: total: calls: 1 - instructions: 980927 + instructions: 987757 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_iter_1k_10kib: total: calls: 1 - instructions: 2465246 + instructions: 2472076 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_iter_20_10mib: total: calls: 1 - instructions: 18468649 + instructions: 18468788 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_iter_rev_1k_0b: total: calls: 1 - instructions: 980453 + instructions: 983283 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_iter_rev_1k_10kib: total: calls: 1 - instructions: 2452399 + instructions: 2455229 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_iter_rev_20_10mib: total: calls: 1 - instructions: 18468641 + instructions: 18468700 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_keys_1k_0b: total: calls: 1 - instructions: 987592 + instructions: 985613 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_keys_1k_10kib: total: calls: 1 - instructions: 2471911 + instructions: 2469932 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_keys_20_10mib: total: calls: 1 - instructions: 18468793 + instructions: 18468754 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_keys_rev_1k_0b: total: calls: 1 - instructions: 986510 + instructions: 984531 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_keys_rev_1k_10kib: total: calls: 1 - instructions: 2458456 + instructions: 2456477 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_keys_rev_20_10mib: total: calls: 1 - instructions: 18468771 + instructions: 18468732 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_values_1k_0b: total: calls: 1 - instructions: 1235927 + instructions: 1242757 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_values_1k_10kib: total: calls: 1 - instructions: 56782402 + instructions: 56789232 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_values_20_10mib: total: calls: 1 - instructions: 1103711407 + instructions: 1103711546 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_values_rev_1k_0b: total: calls: 1 - instructions: 1233785 + instructions: 1236615 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_values_rev_1k_10kib: total: calls: 1 - instructions: 56751789 + instructions: 56754619 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_scan_values_rev_20_10mib: total: calls: 1 - instructions: 1103710947 + instructions: 1103711006 heap_increase: 0 stable_memory_increase: 0 scopes: {} diff --git a/benchmarks/btreemap/src/main.rs b/benchmarks/btreemap/src/main.rs index ff766c85..76b7a227 100644 --- a/benchmarks/btreemap/src/main.rs +++ b/benchmarks/btreemap/src/main.rs @@ -441,6 +441,114 @@ pub fn btreemap_v2_get_10mib_values() -> BenchResult { }) } +// Batch insertion. Each `insert_many` benchmark pairs with the plain-loop benchmark +// directly above it, over exactly the same keys, so the two are directly comparable. + +#[bench(raw)] +pub fn btreemap_v2_insert_many_seq_u64_u64() -> BenchResult { + let mut btree = BTreeMap::new(DefaultMemoryImpl::default()); + let count = 10_000u64; + let mut rng = Rng::from_seed(0); + + // Same keys and values as `btreemap_v2_insert_seq_u64_u64`. + let items: Vec<(u64, u64)> = (0..count).map(|i| (i, rng.rand_u64())).collect(); + bench_fn(|| { + btree.insert_many(items); + }) +} + +#[bench(raw)] +pub fn btreemap_v2_insert_batch_random_u64_u64() -> BenchResult { + let mut btree = BTreeMap::new(DefaultMemoryImpl::default()); + let mut rng = Rng::from_seed(0); + let items = generate_random_kv::(10_000, &mut rng); + bench_fn(|| { + for (key, value) in items { + btree.insert(key, value); + } + }) +} + +// Descending keys. The held path should unwind leftwards just as it unwinds rightwards for +// ascending keys, so this ought to coalesce about as well. +#[bench(raw)] +pub fn btreemap_v2_insert_desc_u64_u64() -> BenchResult { + let mut btree = BTreeMap::new(DefaultMemoryImpl::default()); + let count = 10_000u64; + let mut rng = Rng::from_seed(0); + let items: Vec<(u64, u64)> = (0..count).rev().map(|i| (i, rng.rand_u64())).collect(); + bench_fn(|| { + for (key, value) in items { + btree.insert(key, value); + } + }) +} + +#[bench(raw)] +pub fn btreemap_v2_insert_many_desc_u64_u64() -> BenchResult { + let mut btree = BTreeMap::new(DefaultMemoryImpl::default()); + let count = 10_000u64; + let mut rng = Rng::from_seed(0); + let items: Vec<(u64, u64)> = (0..count).rev().map(|i| (i, rng.rand_u64())).collect(); + bench_fn(|| { + btree.insert_many(items); + }) +} + +// `insert_many` wants ascending keys, so a caller holding unordered data sorts it first. +// The sort is inside the measured section. +#[bench(raw)] +pub fn btreemap_v2_insert_many_sorted_random_u64_u64() -> BenchResult { + let mut btree = BTreeMap::new(DefaultMemoryImpl::default()); + let mut rng = Rng::from_seed(0); + let items = generate_random_kv::(10_000, &mut rng); + bench_fn(|| { + let mut items = items; + items.sort_by(|(a, _), (b, _)| a.cmp(b)); + btree.insert_many(items); + }) +} + +// Feeding unordered keys forfeits the coalescing: the held path unwinds on almost every +// key, so this should land close to the plain insert loop rather than beating it. +#[bench(raw)] +pub fn btreemap_v2_insert_many_unsorted_u64_u64() -> BenchResult { + let mut btree = BTreeMap::new(DefaultMemoryImpl::default()); + let mut rng = Rng::from_seed(0); + let items = generate_random_kv::(10_000, &mut rng); + bench_fn(|| { + btree.insert_many(items); + }) +} + +// Inserting a sorted run into a map that already holds 100k entries: the batch lands in +// leaves that mostly have room, which is where coalescing pays most. +#[bench(raw)] +pub fn btreemap_v2_insert_batch_into_existing_u64_u64() -> BenchResult { + let mut btree = BTreeMap::new(DefaultMemoryImpl::default()); + for i in 0..100_000u64 { + btree.insert(i * 10, i); + } + let items: Vec<(u64, u64)> = (0..10_000u64).map(|i| (i * 10 + 5, i)).collect(); + bench_fn(|| { + for (key, value) in items { + btree.insert(key, value); + } + }) +} + +#[bench(raw)] +pub fn btreemap_v2_insert_many_into_existing_u64_u64() -> BenchResult { + let mut btree = BTreeMap::new(DefaultMemoryImpl::default()); + for i in 0..100_000u64 { + btree.insert(i * 10, i); + } + let items: Vec<(u64, u64)> = (0..10_000u64).map(|i| (i * 10 + 5, i)).collect(); + bench_fn(|| { + btree.insert_many(items); + }) +} + // Benchmarks for `BTreeMap::contains_key`. // Reduced grid: contains_key traversal is identical to get, only skips value deserialization. bench_tests! { diff --git a/src/btreemap.rs b/src/btreemap.rs index afdf6547..48327e96 100644 --- a/src/btreemap.rs +++ b/src/btreemap.rs @@ -49,6 +49,7 @@ //! ---------------------------------------- //! ``` mod allocator; +mod bulk_insert; mod iter; mod node; mod node_cache; @@ -59,6 +60,7 @@ use crate::{ Memory, Storable, }; use allocator::Allocator; +use bulk_insert::BulkInsert; pub use iter::Iter; use node::{DerivedPageSize, Entry, Node, NodeType, PageSize, Version}; use node_cache::NodeCache; @@ -656,8 +658,14 @@ where /// key.to_bytes().len() <= max_size(Key) /// value.to_bytes().len() <= max_size(Value) pub fn insert(&mut self, key: K, value: V) -> Option { - let value = value.into_bytes_checked(); + self.insert_serialized(key, value.into_bytes_checked()) + .map(Cow::Owned) + .map(V::from_bytes) + } + /// The body of [`BTreeMap::insert`], taking an already-serialized value so that + /// callers holding raw bytes (such as [`BTreeMap::insert_many`]) can reuse it. + fn insert_serialized(&mut self, key: K, value: Vec) -> Option> { let root = if self.root_addr == NULL { // No root present. Allocate one. let node = self.allocate_node(NodeType::Leaf); @@ -671,9 +679,7 @@ where // Check if the key already exists in the root. if let Ok(idx) = root.search(&key, self.memory()) { // Key found, replace its value and return the old one. - return Some(V::from_bytes(Cow::Owned( - self.update_value(&mut root, idx, value), - ))); + return Some(self.update_value(&mut root, idx, value)); } // If the root is full, we need to introduce a new node as the root. @@ -702,8 +708,61 @@ where }; self.insert_nonfull(root, key, value, 0) - .map(Cow::Owned) - .map(V::from_bytes) + } + + /// Inserts many key-value pairs, writing each modified node to stable memory at most + /// once instead of once per key. + /// + /// [`insert`](Self::insert) writes the target node — and, when a node splits, its + /// sibling and parent too — on every call, so inserting a run of nearby keys rewrites + /// the same node again and again. `insert_many` instead keeps the current root-to-leaf + /// path in memory and saves a modified node to stable memory only once the input has + /// moved past it. It also writes the header once for the whole call rather than once + /// per key. + /// + /// The iterator is consumed lazily and never collected, so the memory cost is one tree + /// path however many pairs are supplied. + /// + /// Duplicate keys are applied in the order given, so the last occurrence wins, exactly + /// as with repeated `insert` calls — the resulting map is identical, only the number of + /// writes differs. + /// + /// PRECONDITION: for every pair + /// key.to_bytes().len() <= max_size(Key) + /// value.to_bytes().len() <= max_size(Value) + /// + /// # Supply keys in sorted order for best performance + /// + /// The coalescing pays off exactly when consecutive keys land in the same node, so + /// **feed the pairs in sorted key order**. Both ascending and descending work just as + /// well — what matters is that consecutive keys are close together, not which way the + /// run travels. + /// + /// Out-of-order keys are still handled correctly — the path unwinds to an ancestor that + /// covers the key, as far as the root if necessary — but each one costs a fresh + /// descent, and shuffled input performs no better than calling + /// [`insert`](Self::insert) in a loop. Sort first if the data is not already ordered. + /// + /// # Examples + /// + /// ```rust + /// use ic_stable_structures::{BTreeMap, DefaultMemoryImpl}; + /// + /// let mut map: BTreeMap = BTreeMap::new(DefaultMemoryImpl::default()); + /// + /// // Streamed straight from the iterator; nothing is materialized. + /// map.insert_many((0..1_000).map(|i| (i, i * 2))); + /// + /// assert_eq!(map.len(), 1_000); + /// assert_eq!(map.get(&500), Some(1_000)); + /// ``` + pub fn insert_many(&mut self, entries: impl IntoIterator) { + let mut bulk = BulkInsert::new(self); + for (key, value) in entries { + bulk.insert(key, value); + } + // Dropping `bulk` here writes out the path it is still holding, along with the + // header. That is what completes the batch. } /// Inserts an entry into a node that is *not full*. diff --git a/src/btreemap/bulk_insert.rs b/src/btreemap/bulk_insert.rs new file mode 100644 index 00000000..c9bbef7b --- /dev/null +++ b/src/btreemap/bulk_insert.rs @@ -0,0 +1,310 @@ +//! Batched insertion for [`BTreeMap`]. +//! +//! [`BTreeMap::insert_many`] inserts a stream of pairs while holding the current +//! root-to-leaf path in memory, so that each modified node is written to stable memory +//! once the input moves past it rather than once per key. [`BulkInsert`] is that held +//! path, and [`PathLevel`] is one level of it. + +use crate::btreemap::node::{Node, NodeType}; +use crate::types::NULL; +use crate::{BTreeMap, Memory, Storable}; + +/// One level of the root-to-leaf path that [`BTreeMap::insert_many`] holds open across a +/// run of inserts. +struct PathLevel { + node: Node, + + /// Index of this node among its parent's children. Meaningless for the root. + index_in_parent: usize, + + /// The key range this node is responsible for, taken from the separators either side of + /// it while descending. Both ends are exclusive, because a separator key is stored in + /// the parent rather than the child. `None` means unbounded on that side, which is + /// always the case for the root. + /// + /// Both ends are tracked, not just the upper one, so that a key arriving out of order + /// is detected rather than being dropped into the wrong node. See [`PathLevel::covers`]. + lower: Option, + upper: Option, + + /// Distance from the root, needed when handing the node back to the node cache. + depth: u8, + + /// Whether the node has been modified and so must be written out, rather than simply + /// returned to the cache, when it leaves the path. + dirty: bool, +} + +impl PathLevel { + /// Whether `key` belongs somewhere under this node. + /// + /// For an ascending batch only the upper bound can ever fail, and that is the case this + /// is tuned for. The lower bound is checked too so that an out-of-order key unwinds + /// past this node — as far as the root if need be — instead of being inserted here, + /// which would break the ordering invariant of the tree. + fn covers(&self, key: &K) -> bool { + self.lower.as_ref().is_none_or(|lower| lower < key) + && self.upper.as_ref().is_none_or(|upper| key < upper) + } +} + +/// Inserts a stream of keys while keeping the current root-to-leaf path in +/// memory, so that each modified node is written to stable memory once the batch moves +/// past it rather than once per key. +/// +/// Dropping a `BulkInsert` writes back whatever it is still holding: the remaining path, +/// and the header if `length` or `root_addr` moved. That is how a batch is completed, so +/// the drop is load-bearing on every call. +pub(super) struct BulkInsert<'a, K, V, M> +where + K: Storable + Ord + Clone, + V: Storable, + M: Memory, +{ + map: &'a mut BTreeMap, + + /// Root first, leaf last. Empty before the first insert and after a fallback. + path: Vec>, + + /// Whether `length` or `root_addr` changed and the header still needs writing. + header_dirty: bool, +} + +impl<'a, K, V, M> BulkInsert<'a, K, V, M> +where + K: Storable + Ord + Clone, + V: Storable, + M: Memory, +{ + pub(super) fn new(map: &'a mut BTreeMap) -> Self { + Self { + map, + path: Vec::new(), + header_dirty: false, + } + } + + /// Releases a node that has left the path: written out if modified, handed back to the + /// node cache otherwise. + fn release(&mut self, level: PathLevel) { + let mut node = level.node; + if level.dirty { + self.map.save_node(&mut node); + } else { + self.map.return_node(node, level.depth); + } + } + + /// Releases the whole path, leaving it empty. + fn release_path(&mut self) { + while let Some(level) = self.path.pop() { + self.release(level); + } + } + + pub(super) fn insert(&mut self, key: K, value: V) { + // Serialize first, before any node is touched. + let value = value.into_bytes_checked(); + + // Drop back up the path until we reach a node that still covers this key. Since the + // batch ascends, a node stops covering once the keys pass its upper bound. + while self.path.last().is_some_and(|level| !level.covers(&key)) { + let level = self.path.pop().expect("just checked that one exists"); + self.release(level); + } + + if self.path.is_empty() { + self.push_root(); + } + + // Walk down to a leaf, taking each child out of the node cache as we go. + loop { + let bottom = self.path.last().expect("the path always holds a root here"); + if bottom.node.node_type() == NodeType::Leaf { + break; + } + match bottom.node.search(&key, self.map.memory()) { + Ok(idx) => { + // The key lives in this internal node; overwrite it where it sits. + let bottom = self.path.last_mut().expect("checked above"); + bottom.node.swap_value(idx, value, self.map.memory()); + bottom.dirty = true; + return; + } + Err(idx) => self.push_child(idx), + } + } + + self.insert_into_leaf(key, value); + } + + /// Starts a fresh path at the root, allocating one if the map is empty. + fn push_root(&mut self) { + let node = if self.map.root_addr == NULL { + let node = self.map.allocate_node(NodeType::Leaf); + self.map.root_addr = node.address(); + self.header_dirty = true; + node + } else { + self.map.take_or_load_node(self.map.root_addr) + }; + self.path.push(PathLevel { + node, + index_in_parent: 0, + lower: None, + upper: None, + depth: 0, + dirty: false, + }); + } + + /// Descends one level, into the child at `idx` of the node currently at the bottom. + fn push_child(&mut self, idx: usize) { + let (address, lower, upper, depth) = { + let bottom = self.path.last().expect("only called while descending"); + // The separators either side of this child bound the keys it may hold; beyond + // the outermost separator the child inherits its parent's bound. + let lower = if idx > 0 { + Some(bottom.node.key(idx - 1, self.map.memory()).clone()) + } else { + bottom.lower.clone() + }; + let upper = if idx < bottom.node.entries_len() { + Some(bottom.node.key(idx, self.map.memory()).clone()) + } else { + bottom.upper.clone() + }; + ( + bottom.node.child(idx), + lower, + upper, + bottom.depth.saturating_add(1), + ) + }; + let node = self.map.take_or_load_node(address); + self.path.push(PathLevel { + node, + index_in_parent: idx, + lower, + upper, + depth, + dirty: false, + }); + } + + /// Places `key` in the leaf at the bottom of the path, splitting it first if it is + /// full. + fn insert_into_leaf(&mut self, key: K, value: Vec) { + let search = { + let leaf = self.path.last().expect("bottom is a leaf here"); + leaf.node.search(&key, self.map.memory()) + }; + + if let Ok(idx) = search { + let leaf = self.path.last_mut().expect("checked above"); + leaf.node.swap_value(idx, value, self.map.memory()); + leaf.dirty = true; + return; + } + + if self + .path + .last() + .expect("bottom is a leaf here") + .node + .is_full() + && !self.split_leaf(&key) + { + // The split could not be done in place. Write everything back and let the + // ordinary insert path grow the tree, then start a new path for the next key. + self.release_path(); + self.map.insert_serialized(key, value); + self.header_dirty = true; + return; + } + + // A split leaves both halves at the minimum size, so there is room now. + let leaf = self.path.last_mut().expect("bottom is a leaf here"); + let idx = leaf + .node + .search(&key, self.map.memory()) + .expect_err("the key was absent and a split cannot introduce it"); + leaf.node.insert_entry(idx, (key, value)); + leaf.dirty = true; + self.map.length += 1; + self.header_dirty = true; + } + + /// Splits the full leaf at the bottom of the path in two and promotes the median into + /// the parent, leaving the path pointing at whichever half now owns `key`. + /// + /// Returns `false` without touching anything when the split cannot be done here — the + /// leaf is the root, or the parent has no room for the median — leaving the caller to + /// fall back to [`BTreeMap::insert_serialized`], which grows the tree. + fn split_leaf(&mut self, key: &K) -> bool { + let depth_of_path = self.path.len(); + if depth_of_path < 2 || self.path[depth_of_path - 2].node.is_full() { + return false; + } + + let PathLevel { + node: mut left, + index_in_parent, + lower, + upper, + depth, + .. + } = self + .path + .pop() + .expect("checked that the path has a parent level"); + + let mut right = self.map.allocate_node(NodeType::Leaf); + let (median_key, median_value) = left.split(&mut right, self.map.memory()); + + let parent = self + .path + .last_mut() + .expect("checked that the path has a parent level"); + parent + .node + .insert_child(index_in_parent + 1, right.address()); + parent + .node + .insert_entry(index_in_parent, (median_key.clone(), median_value)); + parent.dirty = true; + + // Keep whichever half the key belongs to; the other one is finished with. The key + // cannot equal the median, which was already in the tree while the key was not. + // The median becomes the boundary between the two halves. + let (keep, mut done, keep_index, keep_lower, keep_upper) = if *key < median_key { + (left, right, index_in_parent, lower, Some(median_key)) + } else { + (right, left, index_in_parent + 1, Some(median_key), upper) + }; + self.map.save_node(&mut done); + self.path.push(PathLevel { + node: keep, + index_in_parent: keep_index, + lower: keep_lower, + upper: keep_upper, + depth, + dirty: true, + }); + true + } +} + +impl Drop for BulkInsert<'_, K, V, M> +where + K: Storable + Ord + Clone, + V: Storable, + M: Memory, +{ + fn drop(&mut self) { + self.release_path(); + if self.header_dirty { + self.map.save_header(); + } + } +} diff --git a/src/btreemap/proptests.rs b/src/btreemap/proptests.rs index 8e193823..a8a48cc9 100644 --- a/src/btreemap/proptests.rs +++ b/src/btreemap/proptests.rs @@ -444,3 +444,87 @@ fn execute_operation( } }; } + +// `insert_many` must be indistinguishable from calling `insert` on each pair, whatever the +// batch looks like: unsorted, duplicated, overlapping what is already stored, and whatever +// the node cache is doing. +#[proptest(cases = 30)] +fn insert_many_matches_repeated_insert( + #[strategy(pvec(0..400u64, 0..250))] existing: Vec, + #[strategy(pvec((0..400u64, any::()), 0..400))] batch: Vec<(u64, u64)>, +) { + for cache_slots in [0usize, 1, 16] { + let mut expected: BTreeMap = + BTreeMap::new(make_memory()).with_node_cache(cache_slots); + let mut actual: BTreeMap = + BTreeMap::new(make_memory()).with_node_cache(cache_slots); + + for key in &existing { + expected.insert(*key, *key); + actual.insert(*key, *key); + } + + for (key, value) in &batch { + expected.insert(*key, *value); + } + actual.insert_many(batch.clone()); + + assert_eq!(expected.len(), actual.len(), "cache {cache_slots}"); + let expected_entries: Vec<_> = expected.iter().map(|e| (*e.key(), e.value())).collect(); + let actual_entries: Vec<_> = actual.iter().map(|e| (*e.key(), e.value())).collect(); + assert_eq!(expected_entries, actual_entries, "cache {cache_slots}"); + + // Every key must still be individually reachable, and removable. + for (key, _) in &expected_entries { + assert_eq!(actual.get(key), expected.get(key), "key {key}"); + } + } +} + +// Variable-length keys and values exercise V2 overflow pages and re-serialization, and +// `run_btree_test` covers the V1 and V1-migrated-to-V2 layouts too. +#[proptest(cases = 20)] +fn insert_many_matches_repeated_insert_unbounded( + #[strategy(pvec((pvec(0..8u8, 0..6), pvec(0..255u8, 0..300)), 0..200))] batch: Vec<( + Vec, + Vec, + )>, +) { + run_btree_test(|mut actual| { + let mut expected = StdBTreeMap::new(); + for (key, value) in batch.clone() { + expected.insert(key, value); + } + actual.insert_many(batch.clone()); + + let actual_entries: Vec<_> = actual + .iter() + .map(|e| (e.key().clone(), e.value())) + .collect(); + let expected_entries: Vec<_> = expected + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + prop_assert_eq!(actual_entries, expected_entries); + prop_assert_eq!(actual.len() as usize, expected.len()); + Ok(()) + }); +} + +// Nodes buffered by `insert_many` must all be accounted for: draining the map afterwards +// has to return every chunk to the allocator. +#[proptest(cases = 10)] +fn insert_many_does_not_leak_memory( + #[strategy(pvec((0..2_000u64, any::()), 500..2_000))] batch: Vec<(u64, u64)>, +) { + let mut btree: BTreeMap = BTreeMap::new(make_memory()); + btree.insert_many(batch.clone()); + + let keys: BTreeSet = batch.iter().map(|(k, _)| *k).collect(); + assert_eq!(btree.len() as usize, keys.len()); + for key in keys { + assert!(btree.remove(&key).is_some()); + } + assert!(btree.is_empty()); + assert_eq!(btree.allocator.num_allocated_chunks(), 0); +} diff --git a/src/btreemap/tests.rs b/src/btreemap/tests.rs index 0d7216fc..5e07454b 100644 --- a/src/btreemap/tests.rs +++ b/src/btreemap/tests.rs @@ -3059,3 +3059,252 @@ fn cache_metrics_reset_after_clear() { assert_eq!(after.misses(), 0, "Metrics should reset after clear_new"); assert_eq!(after.total(), 0, "Metrics should reset after clear_new"); } + +// --- insert_many ----------------------------------------------------------- + +/// Builds the same map twice — once with repeated `insert`, once with `insert_many` — and +/// asserts they are indistinguishable. +fn assert_insert_many_matches(existing: &[(u64, u64)], batch: &[(u64, u64)], cache_slots: usize) { + let mut expected: BTreeMap = + BTreeMap::new(make_memory()).with_node_cache(cache_slots); + let mut actual: BTreeMap = + BTreeMap::new(make_memory()).with_node_cache(cache_slots); + + for (key, value) in existing { + expected.insert(*key, *value); + actual.insert(*key, *value); + } + for (key, value) in batch { + expected.insert(*key, *value); + } + actual.insert_many(batch.to_vec()); + + assert_eq!(expected.len(), actual.len(), "cache {cache_slots}"); + assert_eq!( + collect_entry(expected.iter()), + collect_entry(actual.iter()), + "cache {cache_slots}" + ); +} + +#[test] +fn insert_many_empty_batch_is_a_no_op() { + let mut map: BTreeMap = BTreeMap::new(make_memory()); + map.insert_many(Vec::new()); + assert!(map.is_empty()); + + map.insert(1, 1); + map.insert_many(Vec::new()); + assert_eq!(map.len(), 1); + assert_eq!(map.get(&1), Some(1)); +} + +#[test] +fn insert_many_into_empty_map() { + for cache_slots in [0, 1, 16] { + // A single pair has to allocate the root. + assert_insert_many_matches(&[], &[(7, 70)], cache_slots); + // Enough to split the root leaf several times over. + let batch: Vec<(u64, u64)> = (0..500).map(|i| (i, i * 3)).collect(); + assert_insert_many_matches(&[], &batch, cache_slots); + } +} + +#[test] +fn insert_many_orders_and_deduplicates_like_repeated_insert() { + // Descending input must come out sorted. + let descending: Vec<(u64, u64)> = (0..300).rev().map(|i| (i, i)).collect(); + assert_insert_many_matches(&[], &descending, 16); + + // Repeated keys: the last occurrence in the input wins, as with repeated `insert`. + let duplicated = vec![(1, 10), (2, 20), (1, 11), (2, 21), (1, 12)]; + assert_insert_many_matches(&[], &duplicated, 16); + let mut map: BTreeMap = BTreeMap::new(make_memory()); + map.insert_many(duplicated); + assert_eq!(map.get(&1), Some(12)); + assert_eq!(map.get(&2), Some(21)); + assert_eq!(map.len(), 2); +} + +#[test] +fn insert_many_over_existing_entries() { + let existing: Vec<(u64, u64)> = (0..400).map(|i| (i, i)).collect(); + + // Pure overwrite: no key is new, so the length must not move. + let overwrite: Vec<(u64, u64)> = (0..400).map(|i| (i, i + 1_000)).collect(); + assert_insert_many_matches(&existing, &overwrite, 16); + + // Interleaved between existing keys, which fills leaves and forces splits. + let interleaved: Vec<(u64, u64)> = (0..400).map(|i| (i * 2 + 1, i)).collect(); + assert_insert_many_matches(&existing, &interleaved, 16); + + // A run appended past the end, which keeps splitting the rightmost path. + let appended: Vec<(u64, u64)> = (10_000..10_500).map(|i| (i, i)).collect(); + assert_insert_many_matches(&existing, &appended, 16); +} + +#[test] +fn insert_many_exercises_the_split_fallbacks() { + // Dense runs into a map whose leaves are already near capacity make both fallback + // routes fire: a full root leaf, and a parent with no room for a promoted median. + for existing_len in [0u64, 5, 11, 12, 100] { + let existing: Vec<(u64, u64)> = (0..existing_len).map(|i| (i * 100, i)).collect(); + for batch_len in [1u64, 5, 11, 12, 60, 500] { + let batch: Vec<(u64, u64)> = (0..batch_len).map(|i| (i, i)).collect(); + assert_insert_many_matches(&existing, &batch, 0); + assert_insert_many_matches(&existing, &batch, 16); + } + } +} + +#[test] +fn map_is_fully_usable_after_insert_many() { + let mut map: BTreeMap = BTreeMap::new(make_memory()); + map.insert_many((0..1_000u64).map(|i| (i * 2, i))); + + assert_eq!(map.len(), 1_000); + assert_eq!(map.first_key_value(), Some((0, 0))); + assert_eq!(map.last_key_value(), Some((1_998, 999))); + assert_eq!(map.range(10..20).count(), 5); + assert_eq!(map.get(&500), Some(250)); + assert_eq!(map.get(&501), None); + + // Ordinary mutation still works on top of a batch-built tree. + map.insert(501, 12_345); + assert_eq!(map.get(&501), Some(12_345)); + assert_eq!(map.remove(&500), Some(250)); + assert_eq!(map.len(), 1_000); + + for i in 0..1_000u64 { + map.remove(&(i * 2)); + } + map.remove(&501); + assert!(map.is_empty()); + assert_eq!(map.allocator.num_allocated_chunks(), 0); +} + +#[test] +fn insert_many_with_unbounded_values() { + let mut map: BTreeMap, _> = BTreeMap::new(make_memory()); + // Values large enough to spill onto V2 overflow pages. + let batch: Vec<(u64, Vec)> = (0..200u64).map(|i| (i, vec![i as u8; 3_000])).collect(); + map.insert_many(batch.clone()); + + assert_eq!(map.len(), 200); + for (key, value) in &batch { + assert_eq!(map.get(key).as_ref(), Some(value)); + } + + // Overwriting with shorter values through a second batch must reclaim the pages. + map.insert_many((0..200u64).map(|i| (i, vec![i as u8; 2]))); + for i in 0..200u64 { + assert_eq!(map.get(&i), Some(vec![i as u8; 2])); + } + for i in 0..200u64 { + map.remove(&i); + } + assert_eq!(map.allocator.num_allocated_chunks(), 0); +} + +#[test] +fn insert_many_handles_out_of_order_input() { + // `insert_many` is tuned for ascending keys, but wrong ordering must never corrupt the + // tree — it may only cost extra descents. These orderings are chosen to make the path + // unwind constantly, including all the way back to the root. + let n = 1_500u64; + let orderings: Vec<(&str, Vec<(u64, u64)>)> = vec![ + ("descending", (0..n).rev().map(|i| (i, i)).collect()), + ( + // Alternating between the two ends, so every key jumps across the whole tree. + "zigzag", + (0..n) + .map(|i| { + let k = if i % 2 == 0 { i / 2 } else { n - 1 - i / 2 }; + (k, k) + }) + .collect(), + ), + ( + // Ascending runs that repeatedly restart from the beginning. + "sawtooth", + (0..n).map(|i| ((i * 7) % n, i)).collect(), + ), + ( + // Deterministic shuffle. + "shuffled", + { + let mut keys: Vec = (0..n).collect(); + let mut state = 0x2545F4914F6CDD1Du64; + for i in (1..keys.len()).rev() { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + keys.swap(i, (state % (i as u64 + 1)) as usize); + } + keys.into_iter().map(|k| (k, k)).collect() + }, + ), + ]; + + for (label, batch) in orderings { + for cache_slots in [0, 16] { + let mut expected: BTreeMap = + BTreeMap::new(make_memory()).with_node_cache(cache_slots); + let mut actual: BTreeMap = + BTreeMap::new(make_memory()).with_node_cache(cache_slots); + for (key, value) in &batch { + expected.insert(*key, *value); + } + actual.insert_many(batch.clone()); + + assert_eq!(expected.len(), actual.len(), "{label} cache {cache_slots}"); + assert_eq!( + collect_entry(expected.iter()), + collect_entry(actual.iter()), + "{label} cache {cache_slots}" + ); + } + } +} + +#[test] +fn insert_many_streams_without_materializing() { + // The iterator is consumed lazily, so a source far larger than any batch we would want + // to hold in memory still works. `Iterator::map` here is never collected. + let mut map: BTreeMap = BTreeMap::new(make_memory()); + map.insert_many((0..50_000u64).map(|i| (i, i ^ 0xFFFF))); + + assert_eq!(map.len(), 50_000); + assert_eq!(map.get(&0), Some(0xFFFF)); + assert_eq!(map.get(&49_999), Some(49_999 ^ 0xFFFF)); + assert_eq!(map.first_key_value(), Some((0, 0xFFFF))); + assert_eq!(map.last_key_value(), Some((49_999, 49_999 ^ 0xFFFF))); +} + +#[test] +fn insert_many_coalesces_in_either_direction() { + // A descending run is as local as an ascending one — the held path just unwinds + // leftwards — so both must behave the same. Compare against repeated `insert`. + for descending in [false, true] { + let keys: Vec = if descending { + (0..3_000u64).rev().collect() + } else { + (0..3_000u64).collect() + }; + let batch: Vec<(u64, u64)> = keys.iter().map(|k| (*k, k * 3)).collect(); + + let mut expected: BTreeMap = BTreeMap::new(make_memory()); + for (key, value) in &batch { + expected.insert(*key, *value); + } + let mut actual: BTreeMap = BTreeMap::new(make_memory()); + actual.insert_many(batch); + + assert_eq!(expected.len(), actual.len(), "descending={descending}"); + assert_eq!( + collect_entry(expected.iter()), + collect_entry(actual.iter()), + "descending={descending}" + ); + } +} From b6141e3318cfd08385241be7883a924018dfbf6d Mon Sep 17 00:00:00 2001 From: Hamish Peebles Date: Sat, 25 Jul 2026 23:50:05 +0100 Subject: [PATCH 2/5] Improvements --- benchmarks/btreemap/canbench_results.yml | 54 +++++++++++------ benchmarks/btreemap/src/main.rs | 77 +++++++++++++++++++----- src/btreemap/bulk_insert.rs | 7 ++- src/btreemap/node.rs | 7 +++ src/btreemap/tests.rs | 33 ++++++++++ src/btreeset.rs | 65 ++++++++++++++++++++ 6 files changed, 206 insertions(+), 37 deletions(-) diff --git a/benchmarks/btreemap/canbench_results.yml b/benchmarks/btreemap/canbench_results.yml index eb0f84f6..a484ebc1 100644 --- a/benchmarks/btreemap/canbench_results.yml +++ b/benchmarks/btreemap/canbench_results.yml @@ -587,20 +587,6 @@ benches: heap_increase: 161 stable_memory_increase: 3613 scopes: {} - btreemap_v2_insert_batch_into_existing_u64_u64: - total: - calls: 1 - instructions: 294066274 - heap_increase: 0 - stable_memory_increase: 0 - scopes: {} - btreemap_v2_insert_batch_random_u64_u64: - total: - calls: 1 - instructions: 356430975 - heap_increase: 0 - stable_memory_increase: 6 - scopes: {} btreemap_v2_insert_blob8_u64: total: calls: 1 @@ -734,45 +720,73 @@ benches: heap_increase: 0 stable_memory_increase: 20 scopes: {} - btreemap_v2_insert_desc_u64_u64: + btreemap_v2_insert_loop_desc_u64_u64: total: calls: 1 instructions: 386646941 heap_increase: 0 stable_memory_increase: 8 scopes: {} + btreemap_v2_insert_loop_into_existing_u64_u64: + total: + calls: 1 + instructions: 294066274 + heap_increase: 0 + stable_memory_increase: 0 + scopes: {} + btreemap_v2_insert_loop_overwrite_1kib_values: + total: + calls: 1 + instructions: 74252131 + heap_increase: 1 + stable_memory_increase: 0 + scopes: {} + btreemap_v2_insert_loop_random_u64_u64: + total: + calls: 1 + instructions: 356430975 + heap_increase: 0 + stable_memory_increase: 6 + scopes: {} btreemap_v2_insert_many_desc_u64_u64: total: calls: 1 - instructions: 96826813 + instructions: 96975057 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_many_into_existing_u64_u64: total: calls: 1 - instructions: 49506647 + instructions: 49676638 + heap_increase: 0 + stable_memory_increase: 0 + scopes: {} + btreemap_v2_insert_many_overwrite_1kib_values: + total: + calls: 1 + instructions: 10107326 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_insert_many_seq_u64_u64: total: calls: 1 - instructions: 77307535 + instructions: 77454317 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_many_sorted_random_u64_u64: total: calls: 1 - instructions: 86845730 + instructions: 86992512 heap_increase: 3 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_many_unsorted_u64_u64: total: calls: 1 - instructions: 343377265 + instructions: 343413445 heap_increase: 0 stable_memory_increase: 6 scopes: {} diff --git a/benchmarks/btreemap/src/main.rs b/benchmarks/btreemap/src/main.rs index 76b7a227..684ce94e 100644 --- a/benchmarks/btreemap/src/main.rs +++ b/benchmarks/btreemap/src/main.rs @@ -441,8 +441,10 @@ pub fn btreemap_v2_get_10mib_values() -> BenchResult { }) } -// Batch insertion. Each `insert_many` benchmark pairs with the plain-loop benchmark -// directly above it, over exactly the same keys, so the two are directly comparable. +// Batch insertion. Every `insert_many` benchmark has an `insert_loop` counterpart over +// exactly the same keys and values, so the two are directly comparable. The pairs are kept +// adjacent below; the one exception is `insert_many_seq`, whose counterpart is the +// pre-existing `btreemap_v2_insert_seq_u64_u64`. #[bench(raw)] pub fn btreemap_v2_insert_many_seq_u64_u64() -> BenchResult { @@ -457,11 +459,14 @@ pub fn btreemap_v2_insert_many_seq_u64_u64() -> BenchResult { }) } +// Descending keys. The held path should unwind leftwards just as it unwinds rightwards for +// ascending keys, so this ought to coalesce about as well. #[bench(raw)] -pub fn btreemap_v2_insert_batch_random_u64_u64() -> BenchResult { +pub fn btreemap_v2_insert_loop_desc_u64_u64() -> BenchResult { let mut btree = BTreeMap::new(DefaultMemoryImpl::default()); + let count = 10_000u64; let mut rng = Rng::from_seed(0); - let items = generate_random_kv::(10_000, &mut rng); + let items: Vec<(u64, u64)> = (0..count).rev().map(|i| (i, rng.rand_u64())).collect(); bench_fn(|| { for (key, value) in items { btree.insert(key, value); @@ -469,29 +474,27 @@ pub fn btreemap_v2_insert_batch_random_u64_u64() -> BenchResult { }) } -// Descending keys. The held path should unwind leftwards just as it unwinds rightwards for -// ascending keys, so this ought to coalesce about as well. #[bench(raw)] -pub fn btreemap_v2_insert_desc_u64_u64() -> BenchResult { +pub fn btreemap_v2_insert_many_desc_u64_u64() -> BenchResult { let mut btree = BTreeMap::new(DefaultMemoryImpl::default()); let count = 10_000u64; let mut rng = Rng::from_seed(0); let items: Vec<(u64, u64)> = (0..count).rev().map(|i| (i, rng.rand_u64())).collect(); bench_fn(|| { - for (key, value) in items { - btree.insert(key, value); - } + btree.insert_many(items); }) } +// The baseline for both unordered variants below. #[bench(raw)] -pub fn btreemap_v2_insert_many_desc_u64_u64() -> BenchResult { +pub fn btreemap_v2_insert_loop_random_u64_u64() -> BenchResult { let mut btree = BTreeMap::new(DefaultMemoryImpl::default()); - let count = 10_000u64; let mut rng = Rng::from_seed(0); - let items: Vec<(u64, u64)> = (0..count).rev().map(|i| (i, rng.rand_u64())).collect(); + let items = generate_random_kv::(10_000, &mut rng); bench_fn(|| { - btree.insert_many(items); + for (key, value) in items { + btree.insert(key, value); + } }) } @@ -524,7 +527,7 @@ pub fn btreemap_v2_insert_many_unsorted_u64_u64() -> BenchResult { // Inserting a sorted run into a map that already holds 100k entries: the batch lands in // leaves that mostly have room, which is where coalescing pays most. #[bench(raw)] -pub fn btreemap_v2_insert_batch_into_existing_u64_u64() -> BenchResult { +pub fn btreemap_v2_insert_loop_into_existing_u64_u64() -> BenchResult { let mut btree = BTreeMap::new(DefaultMemoryImpl::default()); for i in 0..100_000u64 { btree.insert(i * 10, i); @@ -549,6 +552,50 @@ pub fn btreemap_v2_insert_many_into_existing_u64_u64() -> BenchResult { }) } +// Pure overwrite of existing keys, with values large enough to live on V2 overflow pages. +// `insert` has to read the displaced value back in order to return it; `insert_many` +// discards it and so skips that read entirely. +#[bench(raw)] +pub fn btreemap_v2_insert_loop_overwrite_1kib_values() -> BenchResult { + let count = 2_000usize; + let mut btree = BTreeMap::new(DefaultMemoryImpl::default()); + let mut rng = Rng::from_seed(0); + for (i, value) in generate_random_blocks(count, 1024, &mut rng) + .into_iter() + .enumerate() + { + btree.insert(i as u64, value); + } + let new_values = generate_random_blocks(count, 1024, &mut rng); + bench_fn(|| { + for (i, value) in new_values.into_iter().enumerate() { + btree.insert(i as u64, value); + } + }) +} + +#[bench(raw)] +pub fn btreemap_v2_insert_many_overwrite_1kib_values() -> BenchResult { + let count = 2_000usize; + let mut btree = BTreeMap::new(DefaultMemoryImpl::default()); + let mut rng = Rng::from_seed(0); + for (i, value) in generate_random_blocks(count, 1024, &mut rng) + .into_iter() + .enumerate() + { + btree.insert(i as u64, value); + } + let new_values = generate_random_blocks(count, 1024, &mut rng); + bench_fn(|| { + btree.insert_many( + new_values + .into_iter() + .enumerate() + .map(|(i, v)| (i as u64, v)), + ); + }) +} + // Benchmarks for `BTreeMap::contains_key`. // Reduced grid: contains_key traversal is identical to get, only skips value deserialization. bench_tests! { diff --git a/src/btreemap/bulk_insert.rs b/src/btreemap/bulk_insert.rs index c9bbef7b..9c977f90 100644 --- a/src/btreemap/bulk_insert.rs +++ b/src/btreemap/bulk_insert.rs @@ -127,7 +127,7 @@ where Ok(idx) => { // The key lives in this internal node; overwrite it where it sits. let bottom = self.path.last_mut().expect("checked above"); - bottom.node.swap_value(idx, value, self.map.memory()); + bottom.node.set_value(idx, value); bottom.dirty = true; return; } @@ -202,7 +202,7 @@ where if let Ok(idx) = search { let leaf = self.path.last_mut().expect("checked above"); - leaf.node.swap_value(idx, value, self.map.memory()); + leaf.node.set_value(idx, value); leaf.dirty = true; return; } @@ -219,6 +219,9 @@ where // ordinary insert path grow the tree, then start a new path for the next key. self.release_path(); self.map.insert_serialized(key, value); + // `insert_serialized` saves the header itself on every path that moves `length` + // or `root_addr`, so this only guards against that ceasing to be true. It costs + // at most one extra header write for the whole batch. self.header_dirty = true; return; } diff --git a/src/btreemap/node.rs b/src/btreemap/node.rs index 2efb5cd6..5dae7d9c 100644 --- a/src/btreemap/node.rs +++ b/src/btreemap/node.rs @@ -113,6 +113,13 @@ impl Node { self.entries.len() >= CAPACITY } + /// Replaces the value at `idx`, discarding the old one without reading it. + /// + /// Prefer this to [`Node::swap_value`] when the previous value is not needed. + pub fn set_value(&mut self, idx: usize, new: Vec) { + self.entries[idx].1 = LazyValue::by_value(new); + } + /// Replaces the value at `idx` and returns the old one. pub fn swap_value(&mut self, idx: usize, new: Vec, memory: &M) -> Vec { let old = core::mem::replace(&mut self.entries[idx].1, LazyValue::by_value(new)); diff --git a/src/btreemap/tests.rs b/src/btreemap/tests.rs index 5e07454b..ac969f27 100644 --- a/src/btreemap/tests.rs +++ b/src/btreemap/tests.rs @@ -3206,6 +3206,39 @@ fn insert_many_with_unbounded_values() { assert_eq!(map.allocator.num_allocated_chunks(), 0); } +#[test] +fn insert_many_across_layouts() { + // The V1 node layout differs from V2 in how keys and values are sized and written, and + // `run_btree_test` covers V1, V1-migrated-to-V2 and V2. Bounded types are required + // here: `run_btree_test` silently skips the V1 arms for unbounded ones. + let n = 500u64; + let batches: Vec<(&str, Vec<(u64, u64)>)> = vec![ + ("ascending", (0..n).map(|i| (i, i * 3)).collect()), + ("descending", (0..n).rev().map(|i| (i, i * 3)).collect()), + ("sawtooth", (0..n).map(|i| ((i * 7) % n, i)).collect()), + ("duplicates", vec![(1, 10), (2, 20), (1, 11), (2, 21)]), + ]; + + for (label, batch) in batches { + run_btree_test::(|mut actual| { + let mut expected = std::collections::BTreeMap::new(); + for (key, value) in batch.clone() { + expected.insert(key, value); + } + actual.insert_many(batch.clone()); + + assert_eq!(actual.len() as usize, expected.len(), "{label}"); + let actual_entries: Vec<(u64, u64)> = + actual.iter().map(|e| (*e.key(), e.value())).collect(); + let expected_entries: Vec<(u64, u64)> = expected.into_iter().collect(); + assert_eq!(actual_entries, expected_entries, "{label}"); + for (key, value) in &expected_entries { + assert_eq!(actual.get(key), Some(*value), "{label} key {key}"); + } + }); + } +} + #[test] fn insert_many_handles_out_of_order_input() { // `insert_many` is tuned for ascending keys, but wrong ordering must never corrupt the diff --git a/src/btreeset.rs b/src/btreeset.rs index d4853b15..8f4f924f 100644 --- a/src/btreeset.rs +++ b/src/btreeset.rs @@ -256,6 +256,29 @@ where self.map.insert(key, ()).is_none() } + /// Inserts many keys, writing each modified node to stable memory at most once instead + /// of once per key. See [`BTreeMap::insert_many`] for the details. + /// + /// Supply the keys in sorted order — ascending or descending — for the coalescing to + /// pay off. Out-of-order keys are handled correctly but forfeit the speedup. + /// + /// The iterator is consumed lazily and never collected. + /// + /// # Example + /// + /// ```rust + /// use ic_stable_structures::{BTreeSet, DefaultMemoryImpl}; + /// + /// let mut set: BTreeSet = BTreeSet::new(DefaultMemoryImpl::default()); + /// set.insert_many(0..1_000); + /// + /// assert_eq!(set.len(), 1_000); + /// assert!(set.contains(&500)); + /// ``` + pub fn insert_many(&mut self, keys: impl IntoIterator) { + self.map.insert_many(keys.into_iter().map(|key| (key, ()))); + } + /// Returns `true` if the key exists in the set, `false` otherwise. /// /// # Complexity @@ -1249,6 +1272,48 @@ mod test { assert!(range.is_empty()); } + #[test] + fn test_insert_many_matches_repeated_insert() { + // Ascending, descending, unordered and duplicated inputs must all leave the set in + // exactly the state repeated `insert` calls would. + let n = 1_000u32; + let batches: Vec<(&str, Vec)> = vec![ + ("ascending", (0..n).collect()), + ("descending", (0..n).rev().collect()), + ("sawtooth", (0..n).map(|i| (i * 7) % n).collect()), + ("duplicates", vec![1, 2, 1, 3, 2, 1]), + ]; + + for (label, batch) in batches { + let mut expected: BTreeSet = BTreeSet::new(make_memory()); + for key in &batch { + expected.insert(*key); + } + let mut actual: BTreeSet = BTreeSet::new(make_memory()); + actual.insert_many(batch.clone()); + + assert_eq!(actual.len(), expected.len(), "{label}"); + let actual_keys: Vec = actual.iter().collect(); + let expected_keys: Vec = expected.iter().collect(); + assert_eq!(actual_keys, expected_keys, "{label}"); + for key in &expected_keys { + assert!(actual.contains(key), "{label} key {key}"); + } + } + } + + #[test] + fn test_insert_many_empty_batch_is_a_no_op() { + let mut btreeset: BTreeSet = BTreeSet::new(make_memory()); + btreeset.insert_many(Vec::new()); + assert!(btreeset.is_empty()); + + btreeset.insert(1); + btreeset.insert_many(Vec::new()); + assert_eq!(btreeset.len(), 1); + assert!(btreeset.contains(&1)); + } + #[test] fn test_insert_and_remove_large_set() { let mem = make_memory(); From 67fa2bcb81257221c5589976d13fc591968dea56 Mon Sep 17 00:00:00 2001 From: Hamish Peebles Date: Sun, 26 Jul 2026 00:04:25 +0100 Subject: [PATCH 3/5] Clean it up a bit --- src/btreemap.rs | 14 ++++++-------- src/btreemap/bulk_insert.rs | 23 +++++++++++------------ 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/btreemap.rs b/src/btreemap.rs index 48327e96..9c5a630a 100644 --- a/src/btreemap.rs +++ b/src/btreemap.rs @@ -658,14 +658,8 @@ where /// key.to_bytes().len() <= max_size(Key) /// value.to_bytes().len() <= max_size(Value) pub fn insert(&mut self, key: K, value: V) -> Option { - self.insert_serialized(key, value.into_bytes_checked()) - .map(Cow::Owned) - .map(V::from_bytes) - } + let value = value.into_bytes_checked(); - /// The body of [`BTreeMap::insert`], taking an already-serialized value so that - /// callers holding raw bytes (such as [`BTreeMap::insert_many`]) can reuse it. - fn insert_serialized(&mut self, key: K, value: Vec) -> Option> { let root = if self.root_addr == NULL { // No root present. Allocate one. let node = self.allocate_node(NodeType::Leaf); @@ -679,7 +673,9 @@ where // Check if the key already exists in the root. if let Ok(idx) = root.search(&key, self.memory()) { // Key found, replace its value and return the old one. - return Some(self.update_value(&mut root, idx, value)); + return Some(V::from_bytes(Cow::Owned( + self.update_value(&mut root, idx, value), + ))); } // If the root is full, we need to introduce a new node as the root. @@ -708,6 +704,8 @@ where }; self.insert_nonfull(root, key, value, 0) + .map(Cow::Owned) + .map(V::from_bytes) } /// Inserts many key-value pairs, writing each modified node to stable memory at most diff --git a/src/btreemap/bulk_insert.rs b/src/btreemap/bulk_insert.rs index 9c977f90..243daa32 100644 --- a/src/btreemap/bulk_insert.rs +++ b/src/btreemap/bulk_insert.rs @@ -103,9 +103,6 @@ where } pub(super) fn insert(&mut self, key: K, value: V) { - // Serialize first, before any node is touched. - let value = value.into_bytes_checked(); - // Drop back up the path until we reach a node that still covers this key. Since the // batch ascends, a node stops covering once the keys pass its upper bound. while self.path.last().is_some_and(|level| !level.covers(&key)) { @@ -127,7 +124,7 @@ where Ok(idx) => { // The key lives in this internal node; overwrite it where it sits. let bottom = self.path.last_mut().expect("checked above"); - bottom.node.set_value(idx, value); + bottom.node.set_value(idx, value.into_bytes_checked()); bottom.dirty = true; return; } @@ -194,7 +191,7 @@ where /// Places `key` in the leaf at the bottom of the path, splitting it first if it is /// full. - fn insert_into_leaf(&mut self, key: K, value: Vec) { + fn insert_into_leaf(&mut self, key: K, value: V) { let search = { let leaf = self.path.last().expect("bottom is a leaf here"); leaf.node.search(&key, self.map.memory()) @@ -202,7 +199,7 @@ where if let Ok(idx) = search { let leaf = self.path.last_mut().expect("checked above"); - leaf.node.set_value(idx, value); + leaf.node.set_value(idx, value.into_bytes_checked()); leaf.dirty = true; return; } @@ -217,11 +214,12 @@ where { // The split could not be done in place. Write everything back and let the // ordinary insert path grow the tree, then start a new path for the next key. + // The key is absent, so nothing is deserialized to build the discarded return. self.release_path(); - self.map.insert_serialized(key, value); - // `insert_serialized` saves the header itself on every path that moves `length` - // or `root_addr`, so this only guards against that ceasing to be true. It costs - // at most one extra header write for the whole batch. + self.map.insert(key, value); + // `insert` saves the header itself on every path that moves `length` or + // `root_addr`, so this only guards against that ceasing to be true. It costs at + // most one extra header write for the whole batch. self.header_dirty = true; return; } @@ -232,7 +230,8 @@ where .node .search(&key, self.map.memory()) .expect_err("the key was absent and a split cannot introduce it"); - leaf.node.insert_entry(idx, (key, value)); + leaf.node + .insert_entry(idx, (key, value.into_bytes_checked())); leaf.dirty = true; self.map.length += 1; self.header_dirty = true; @@ -243,7 +242,7 @@ where /// /// Returns `false` without touching anything when the split cannot be done here — the /// leaf is the root, or the parent has no room for the median — leaving the caller to - /// fall back to [`BTreeMap::insert_serialized`], which grows the tree. + /// fall back to [`BTreeMap::insert`], which grows the tree. fn split_leaf(&mut self, key: &K) -> bool { let depth_of_path = self.path.len(); if depth_of_path < 2 || self.path[depth_of_path - 2].node.is_full() { From d314962eab48df8cb560185081e57ea456a1129f Mon Sep 17 00:00:00 2001 From: Hamish Peebles Date: Sun, 26 Jul 2026 00:10:29 +0100 Subject: [PATCH 4/5] Update benchmarks --- benchmarks/btreemap/canbench_results.yml | 150 +++++++++++------------ 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/benchmarks/btreemap/canbench_results.yml b/benchmarks/btreemap/canbench_results.yml index a484ebc1..88bde88f 100644 --- a/benchmarks/btreemap/canbench_results.yml +++ b/benchmarks/btreemap/canbench_results.yml @@ -184,7 +184,7 @@ benches: btreemap_v2_get_blob8_u64: total: calls: 1 - instructions: 206168780 + instructions: 205616407 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -289,7 +289,7 @@ benches: btreemap_v2_get_blob_4_128: total: calls: 1 - instructions: 169374118 + instructions: 169181540 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -345,21 +345,21 @@ benches: btreemap_v2_get_then_insert_blob_32_128: total: calls: 1 - instructions: 525735862 + instructions: 525765859 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_get_then_insert_u64_u64: total: calls: 1 - instructions: 375267793 + instructions: 375037787 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_get_then_insert_u64_u64_nocache: total: calls: 1 - instructions: 604814288 + instructions: 604584122 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -408,7 +408,7 @@ benches: btreemap_v2_get_vec8_u64: total: calls: 1 - instructions: 268419835 + instructions: 265721428 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -576,413 +576,413 @@ benches: btreemap_v2_insert_100k_u64_u64: total: calls: 1 - instructions: 4294470481 + instructions: 4291970481 heap_increase: 0 stable_memory_increase: 60 scopes: {} btreemap_v2_insert_10mib_values: total: calls: 1 - instructions: 4370305256 + instructions: 4370305296 heap_increase: 161 stable_memory_increase: 3613 scopes: {} btreemap_v2_insert_blob8_u64: total: calls: 1 - instructions: 343422096 + instructions: 342846163 heap_increase: 0 stable_memory_increase: 4 scopes: {} btreemap_v2_insert_blob_1024_128: total: calls: 1 - instructions: 4089409188 + instructions: 4089419201 heap_increase: 2 stable_memory_increase: 196 scopes: {} btreemap_v2_insert_blob_128_128: total: calls: 1 - instructions: 898928072 + instructions: 898938251 heap_increase: 0 stable_memory_increase: 46 scopes: {} btreemap_v2_insert_blob_16_128: total: calls: 1 - instructions: 397221173 + instructions: 397233643 heap_increase: 0 stable_memory_increase: 24 scopes: {} btreemap_v2_insert_blob_256_128: total: calls: 1 - instructions: 1385774087 + instructions: 1385784168 heap_increase: 0 stable_memory_increase: 67 scopes: {} btreemap_v2_insert_blob_32_0: total: calls: 1 - instructions: 403980756 + instructions: 404011679 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_blob_32_1024: total: calls: 1 - instructions: 640524429 + instructions: 640535326 heap_increase: 0 stable_memory_increase: 173 scopes: {} btreemap_v2_insert_blob_32_128: total: calls: 1 - instructions: 448735453 + instructions: 448746352 heap_increase: 0 stable_memory_increase: 28 scopes: {} btreemap_v2_insert_blob_32_16: total: calls: 1 - instructions: 438414948 + instructions: 438405801 heap_increase: 0 stable_memory_increase: 11 scopes: {} btreemap_v2_insert_blob_32_256: total: calls: 1 - instructions: 479109269 + instructions: 479120096 heap_increase: 0 stable_memory_increase: 49 scopes: {} btreemap_v2_insert_blob_32_32: total: calls: 1 - instructions: 433824242 + instructions: 433815169 heap_increase: 0 stable_memory_increase: 13 scopes: {} btreemap_v2_insert_blob_32_4: total: calls: 1 - instructions: 420408336 + instructions: 420419229 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_blob_32_512: total: calls: 1 - instructions: 543365153 + instructions: 543376026 heap_increase: 0 stable_memory_increase: 91 scopes: {} btreemap_v2_insert_blob_32_64: total: calls: 1 - instructions: 447289930 + instructions: 447280905 heap_increase: 0 stable_memory_increase: 18 scopes: {} btreemap_v2_insert_blob_32_8: total: calls: 1 - instructions: 423937590 + instructions: 423948483 heap_increase: 0 stable_memory_increase: 9 scopes: {} btreemap_v2_insert_blob_4_128: total: calls: 1 - instructions: 325959134 + instructions: 325528568 heap_increase: 0 stable_memory_increase: 13 scopes: {} btreemap_v2_insert_blob_512_128: total: calls: 1 - instructions: 2240835899 + instructions: 2240845938 heap_increase: 1 stable_memory_increase: 111 scopes: {} btreemap_v2_insert_blob_64_128: total: calls: 1 - instructions: 565730841 + instructions: 565741268 heap_increase: 0 stable_memory_increase: 34 scopes: {} btreemap_v2_insert_blob_8_128: total: calls: 1 - instructions: 370608603 + instructions: 370623180 heap_increase: 0 stable_memory_increase: 20 scopes: {} btreemap_v2_insert_loop_desc_u64_u64: total: calls: 1 - instructions: 386646941 + instructions: 386396941 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_loop_into_existing_u64_u64: total: calls: 1 - instructions: 294066274 + instructions: 293816274 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_insert_loop_overwrite_1kib_values: total: calls: 1 - instructions: 74252131 + instructions: 74288211 heap_increase: 1 stable_memory_increase: 0 scopes: {} btreemap_v2_insert_loop_random_u64_u64: total: calls: 1 - instructions: 356430975 + instructions: 356180975 heap_increase: 0 stable_memory_increase: 6 scopes: {} btreemap_v2_insert_many_desc_u64_u64: total: calls: 1 - instructions: 96975057 + instructions: 96583320 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_many_into_existing_u64_u64: total: calls: 1 - instructions: 49676638 + instructions: 48936213 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_insert_many_overwrite_1kib_values: total: calls: 1 - instructions: 10107326 + instructions: 10081750 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_insert_many_seq_u64_u64: total: calls: 1 - instructions: 77454317 + instructions: 77168556 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_many_sorted_random_u64_u64: total: calls: 1 - instructions: 86992512 + instructions: 86706751 heap_increase: 3 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_many_unsorted_u64_u64: total: calls: 1 - instructions: 343413445 + instructions: 341011825 heap_increase: 0 stable_memory_increase: 6 scopes: {} btreemap_v2_insert_overwrite_u64_u64: total: calls: 1 - instructions: 296294757 + instructions: 296064753 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_insert_overwrite_u64_u64_nocache: total: calls: 1 - instructions: 371676272 + instructions: 371446106 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_insert_overwrite_zipf_10k_u64_u64: total: calls: 1 - instructions: 265316225 + instructions: 265063374 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_insert_principal: total: calls: 1 - instructions: 397834639 + instructions: 397804971 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_seq_u64_u64: total: calls: 1 - instructions: 334191577 + instructions: 333941577 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_seq_u64_u64_nocache: total: calls: 1 - instructions: 473001467 + instructions: 472751467 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_u64_blob8: total: calls: 1 - instructions: 352851122 + instructions: 352841122 heap_increase: 0 stable_memory_increase: 5 scopes: {} btreemap_v2_insert_u64_u64: total: calls: 1 - instructions: 356430975 + instructions: 356180975 heap_increase: 0 stable_memory_increase: 6 scopes: {} btreemap_v2_insert_u64_vec8: total: calls: 1 - instructions: 351036169 + instructions: 351056169 heap_increase: 0 stable_memory_increase: 21 scopes: {} btreemap_v2_insert_vec8_u64: total: calls: 1 - instructions: 434503480 + instructions: 434940532 heap_increase: 0 stable_memory_increase: 16 scopes: {} btreemap_v2_insert_vec_1024_128: total: calls: 1 - instructions: 2194429661 + instructions: 2194439661 heap_increase: 0 stable_memory_increase: 193 scopes: {} btreemap_v2_insert_vec_128_128: total: calls: 1 - instructions: 864135973 + instructions: 864145973 heap_increase: 0 stable_memory_increase: 51 scopes: {} btreemap_v2_insert_vec_16_128: total: calls: 1 - instructions: 563873345 + instructions: 563883375 heap_increase: 0 stable_memory_increase: 31 scopes: {} btreemap_v2_insert_vec_256_128: total: calls: 1 - instructions: 1183331041 + instructions: 1183341041 heap_increase: 0 stable_memory_increase: 71 scopes: {} btreemap_v2_insert_vec_32_0: total: calls: 1 - instructions: 510952957 + instructions: 510924354 heap_increase: 0 stable_memory_increase: 20 scopes: {} btreemap_v2_insert_vec_32_1024: total: calls: 1 - instructions: 1050297481 + instructions: 1050307481 heap_increase: 0 stable_memory_increase: 171 scopes: {} btreemap_v2_insert_vec_32_128: total: calls: 1 - instructions: 635239590 + instructions: 635249590 heap_increase: 0 stable_memory_increase: 33 scopes: {} btreemap_v2_insert_vec_32_16: total: calls: 1 - instructions: 562766441 + instructions: 562776441 heap_increase: 0 stable_memory_increase: 20 scopes: {} btreemap_v2_insert_vec_32_256: total: calls: 1 - instructions: 739231709 + instructions: 739241709 heap_increase: 0 stable_memory_increase: 54 scopes: {} btreemap_v2_insert_vec_32_32: total: calls: 1 - instructions: 556976479 + instructions: 556986479 heap_increase: 0 stable_memory_increase: 20 scopes: {} btreemap_v2_insert_vec_32_4: total: calls: 1 - instructions: 539333255 + instructions: 539343255 heap_increase: 0 stable_memory_increase: 20 scopes: {} btreemap_v2_insert_vec_32_512: total: calls: 1 - instructions: 835651222 + instructions: 835661222 heap_increase: 0 stable_memory_increase: 91 scopes: {} btreemap_v2_insert_vec_32_64: total: calls: 1 - instructions: 564249167 + instructions: 564259167 heap_increase: 0 stable_memory_increase: 24 scopes: {} btreemap_v2_insert_vec_32_8: total: calls: 1 - instructions: 533040455 + instructions: 533050455 heap_increase: 0 stable_memory_increase: 20 scopes: {} btreemap_v2_insert_vec_4_128: total: calls: 1 - instructions: 436519587 + instructions: 436530007 heap_increase: 1 stable_memory_increase: 16 scopes: {} btreemap_v2_insert_vec_512_128: total: calls: 1 - instructions: 1570135282 + instructions: 1570145282 heap_increase: 0 stable_memory_increase: 112 scopes: {} btreemap_v2_insert_vec_64_128: total: calls: 1 - instructions: 722888819 + instructions: 722898819 heap_increase: 0 stable_memory_increase: 41 scopes: {} btreemap_v2_insert_vec_8_128: total: calls: 1 - instructions: 528253615 + instructions: 528263675 heap_increase: 0 stable_memory_increase: 23 scopes: {} @@ -1087,35 +1087,35 @@ benches: btreemap_v2_mem_manager_insert_blob512_u64: total: calls: 1 - instructions: 2319424809 + instructions: 2319434856 heap_increase: 1 stable_memory_increase: 0 scopes: {} btreemap_v2_mem_manager_insert_u64_blob512: total: calls: 1 - instructions: 534143599 + instructions: 534133599 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mem_manager_insert_u64_u64: total: calls: 1 - instructions: 434555466 + instructions: 434545466 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mem_manager_insert_u64_vec512: total: calls: 1 - instructions: 736567925 + instructions: 736587925 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mem_manager_insert_vec512_u64: total: calls: 1 - instructions: 1619612768 + instructions: 1619562818 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1157,14 +1157,14 @@ benches: btreemap_v2_mixed_get_insert_zipf_10k_u64_u64: total: calls: 1 - instructions: 337669142 + instructions: 337385316 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_v2_mixed_get_insert_zipf_10k_u64_u64_nocache: total: calls: 1 - instructions: 576288777 + instructions: 576049647 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1409,7 +1409,7 @@ benches: btreemap_v2_remove_blob8_u64: total: calls: 1 - instructions: 494421991 + instructions: 494526895 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1514,7 +1514,7 @@ benches: btreemap_v2_remove_blob_4_128: total: calls: 1 - instructions: 373429303 + instructions: 373323185 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -1570,7 +1570,7 @@ benches: btreemap_v2_remove_vec8_u64: total: calls: 1 - instructions: 639504895 + instructions: 638942966 heap_increase: 0 stable_memory_increase: 0 scopes: {} From 789121594ebab4008701dd1ce007c49cc614b110 Mon Sep 17 00:00:00 2001 From: Hamish Peebles Date: Sun, 26 Jul 2026 11:16:50 +0100 Subject: [PATCH 5/5] perf(btreemap): cascade splits in place during `insert_many` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A leaf split promotes a median into its parent, so a full parent used to end the batch: the path was flushed and the ordinary insert path grew the tree instead. Every node the path had buffered was then written again on the way back down, which cost roughly half of the writes `insert_many` was saving. The split now cascades. `split_leaf` walks up to the topmost full level, grows a new root if the whole path is full, then splits top-down: each split leaves its node half empty, so the level below always has room for the median it promotes. Writing each dirty node exactly once is now reached rather than approached — 10k ascending keys perform 1,997 writes, matching the number of distinct nodes in the resulting tree, down from 4,032. An `insert` loop over the same keys performs 15,976. ascending, empty map 77.17M -> 35.88M instructions (4.3x -> 9.3x) unordered, caller sorts 86.71M -> 45.42M instructions (4.1x -> 7.8x) descending, empty map 96.58M -> 52.53M instructions (4.0x -> 7.4x) Nothing else moves: 262 benchmarks, 0 regressed, 3 improved. The workloads that barely split — inserting into gaps in an existing map, pure overwrites, unordered input — are unchanged, which is where the saving was expected to come from. The levels below a split keep the bounds they already hold: the separators either side of them survive, landing in one half or the other. Only their index among the parent's children moves, by the number of children that stayed with the left half. Removing the fallback also removes the last call from `BulkInsert` back into `BTreeMap::insert`. Co-Authored-By: Claude Opus 5 --- benchmarks/btreemap/canbench_results.yml | 10 +- src/btreemap/bulk_insert.rs | 148 ++++++++++++++--------- src/btreemap/tests.rs | 48 +++++++- 3 files changed, 143 insertions(+), 63 deletions(-) diff --git a/benchmarks/btreemap/canbench_results.yml b/benchmarks/btreemap/canbench_results.yml index 88bde88f..7dfed3e3 100644 --- a/benchmarks/btreemap/canbench_results.yml +++ b/benchmarks/btreemap/canbench_results.yml @@ -751,14 +751,14 @@ benches: btreemap_v2_insert_many_desc_u64_u64: total: calls: 1 - instructions: 96583320 + instructions: 52532247 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_many_into_existing_u64_u64: total: calls: 1 - instructions: 48936213 + instructions: 49329549 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -772,21 +772,21 @@ benches: btreemap_v2_insert_many_seq_u64_u64: total: calls: 1 - instructions: 77168556 + instructions: 35883157 heap_increase: 0 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_many_sorted_random_u64_u64: total: calls: 1 - instructions: 86706751 + instructions: 45421352 heap_increase: 3 stable_memory_increase: 8 scopes: {} btreemap_v2_insert_many_unsorted_u64_u64: total: calls: 1 - instructions: 341011825 + instructions: 336928720 heap_increase: 0 stable_memory_increase: 6 scopes: {} diff --git a/src/btreemap/bulk_insert.rs b/src/btreemap/bulk_insert.rs index 243daa32..5b7352ef 100644 --- a/src/btreemap/bulk_insert.rs +++ b/src/btreemap/bulk_insert.rs @@ -63,7 +63,7 @@ where { map: &'a mut BTreeMap, - /// Root first, leaf last. Empty before the first insert and after a fallback. + /// Root first, leaf last. Empty only before the first insert. path: Vec>, /// Whether `length` or `root_addr` changed and the header still needs writing. @@ -210,18 +210,8 @@ where .expect("bottom is a leaf here") .node .is_full() - && !self.split_leaf(&key) { - // The split could not be done in place. Write everything back and let the - // ordinary insert path grow the tree, then start a new path for the next key. - // The key is absent, so nothing is deserialized to build the discarded return. - self.release_path(); - self.map.insert(key, value); - // `insert` saves the header itself on every path that moves `length` or - // `root_addr`, so this only guards against that ceasing to be true. It costs at - // most one extra header write for the whole batch. - self.header_dirty = true; - return; + self.split_leaf(&key); } // A split leaves both halves at the minimum size, so there is room now. @@ -237,40 +227,76 @@ where self.header_dirty = true; } - /// Splits the full leaf at the bottom of the path in two and promotes the median into - /// the parent, leaving the path pointing at whichever half now owns `key`. + /// Makes room for `key` in the full leaf at the bottom of the path. /// - /// Returns `false` without touching anything when the split cannot be done here — the - /// leaf is the root, or the parent has no room for the median — leaving the caller to - /// fall back to [`BTreeMap::insert`], which grows the tree. - fn split_leaf(&mut self, key: &K) -> bool { - let depth_of_path = self.path.len(); - if depth_of_path < 2 || self.path[depth_of_path - 2].node.is_full() { - return false; + /// A leaf split promotes a median into the parent, which may itself be full, so the + /// split cascades up the path through every full ancestor — growing a new root if the + /// whole path is full. Afterwards the bottom of the path is the half that owns `key`, + /// with room for it. + fn split_leaf(&mut self, key: &K) { + // Find the topmost level that has to split. Everything from there down to the leaf + // is full, so none of them has anywhere to promote a median until the level above + // it has split and made room. + let mut topmost = self.path.len() - 1; + while topmost > 0 && self.path[topmost - 1].node.is_full() { + topmost -= 1; } - let PathLevel { - node: mut left, - index_in_parent, - lower, - upper, - depth, - .. - } = self - .path - .pop() - .expect("checked that the path has a parent level"); + if topmost == 0 { + // The root is full too, so the tree gains a level and everything already on the + // path moves one deeper. + self.grow_root(); + topmost = 1; + } - let mut right = self.map.allocate_node(NodeType::Leaf); - let (median_key, median_value) = left.split(&mut right, self.map.memory()); + // Split downwards. Each split leaves its node half empty, so by the time the level + // below it splits, there is room for the median it promotes. + for level in topmost..self.path.len() { + self.split_level(level, key); + } + } - let parent = self - .path - .last_mut() - .expect("checked that the path has a parent level"); - parent - .node - .insert_child(index_in_parent + 1, right.address()); + /// Inserts a fresh internal root above the current one, leaving the tree a level deeper + /// and the path a level longer. The new root holds the old one as its only child, and + /// so has room for the median about to be promoted into it. + fn grow_root(&mut self) { + let old_root = self.path[0].node.address(); + let mut root = self.map.allocate_node(NodeType::Internal); + root.push_child(old_root); + self.map.root_addr = root.address(); + self.header_dirty = true; + + for level in self.path.iter_mut() { + level.depth = level.depth.saturating_add(1); + } + self.path.insert( + 0, + PathLevel { + node: root, + index_in_parent: 0, + lower: None, + upper: None, + depth: 0, + dirty: true, + }, + ); + } + + /// Splits the full node at `path[index]` in two and promotes the median into its + /// parent, which must have room. The path keeps whichever half owns `key`; the other + /// half is finished with and is written out. + fn split_level(&mut self, index: usize, key: &K) { + debug_assert!(index > 0, "the root splits only after `grow_root`"); + debug_assert!(!self.path[index - 1].node.is_full()); + debug_assert!(self.path[index].node.is_full()); + + let mut right = self.map.allocate_node(self.path[index].node.node_type()); + let right_addr = right.address(); + let (median_key, median_value) = self.path[index].node.split(&mut right, self.map.memory()); + let index_in_parent = self.path[index].index_in_parent; + + let parent = &mut self.path[index - 1]; + parent.node.insert_child(index_in_parent + 1, right_addr); parent .node .insert_entry(index_in_parent, (median_key.clone(), median_value)); @@ -279,21 +305,33 @@ where // Keep whichever half the key belongs to; the other one is finished with. The key // cannot equal the median, which was already in the tree while the key was not. // The median becomes the boundary between the two halves. - let (keep, mut done, keep_index, keep_lower, keep_upper) = if *key < median_key { - (left, right, index_in_parent, lower, Some(median_key)) + // + // The levels below keep the bounds they already have: the separators either side of + // them survive the split, they just end up in one half or the other. + if *key < median_key { + // The left half stays where it is, since `split` left it in place. + let level = &mut self.path[index]; + level.upper = Some(median_key); + level.dirty = true; + self.map.save_node(&mut right); } else { - (right, left, index_in_parent + 1, Some(median_key), upper) - }; - self.map.save_node(&mut done); - self.path.push(PathLevel { - node: keep, - index_in_parent: keep_index, - lower: keep_lower, - upper: keep_upper, - depth, - dirty: true, - }); - true + let (mut left, moved_children) = { + let level = &mut self.path[index]; + let left = core::mem::replace(&mut level.node, right); + level.index_in_parent = index_in_parent + 1; + level.lower = Some(median_key); + level.dirty = true; + let moved_children = left.children_len(); + (left, moved_children) + }; + self.map.save_node(&mut left); + + // The level below travelled with the entries into the right half, so it sits at + // a lower index among its parent's children than it did before. + if let Some(below) = self.path.get_mut(index + 1) { + below.index_in_parent -= moved_children; + } + } } } diff --git a/src/btreemap/tests.rs b/src/btreemap/tests.rs index ac969f27..58bc8184 100644 --- a/src/btreemap/tests.rs +++ b/src/btreemap/tests.rs @@ -3144,9 +3144,10 @@ fn insert_many_over_existing_entries() { } #[test] -fn insert_many_exercises_the_split_fallbacks() { - // Dense runs into a map whose leaves are already near capacity make both fallback - // routes fire: a full root leaf, and a parent with no room for a promoted median. +fn insert_many_exercises_the_split_cascade() { + // Dense runs into a map whose leaves are already near capacity drive both cascade + // routes: a full root, and a parent with no room for a promoted median. The batch + // sizes straddle the node capacity of 11 so that splits land on the boundaries. for existing_len in [0u64, 5, 11, 12, 100] { let existing: Vec<(u64, u64)> = (0..existing_len).map(|i| (i * 100, i)).collect(); for batch_len in [1u64, 5, 11, 12, 60, 500] { @@ -3157,6 +3158,47 @@ fn insert_many_exercises_the_split_fallbacks() { } } +#[test] +fn insert_many_grows_the_tree_through_repeated_root_splits() { + // Enough keys to push the root down several levels, so the cascade has to grow a new + // root more than once and re-index the levels below each split it performs. Draining + // the map afterwards is the structural check: a mislinked child or a stale index would + // strand nodes and leave the allocator holding chunks. + for descending in [false, true] { + for cache_slots in [0, 16] { + let n = 20_000u64; + let keys: Vec = if descending { + (0..n).rev().collect() + } else { + (0..n).collect() + }; + + let mut map: BTreeMap = + BTreeMap::new(make_memory()).with_node_cache(cache_slots); + map.insert_many(keys.iter().map(|k| (*k, k * 2))); + + let label = format!("descending={descending} cache={cache_slots}"); + assert_eq!(map.len(), n, "{label}"); + assert_eq!(map.first_key_value(), Some((0, 0)), "{label}"); + assert_eq!(map.last_key_value(), Some((n - 1, (n - 1) * 2)), "{label}"); + + // Every key readable, and in order. + let entries = collect_entry(map.iter()); + assert_eq!(entries.len(), n as usize, "{label}"); + for (i, (key, value)) in entries.into_iter().enumerate() { + assert_eq!(key, i as u64, "{label}"); + assert_eq!(value, i as u64 * 2, "{label}"); + } + + for key in 0..n { + assert_eq!(map.remove(&key), Some(key * 2), "{label} key {key}"); + } + assert!(map.is_empty(), "{label}"); + assert_eq!(map.allocator.num_allocated_chunks(), 0, "{label}"); + } + } +} + #[test] fn map_is_fully_usable_after_insert_many() { let mut map: BTreeMap = BTreeMap::new(make_memory());